[Online quantization] Add targeted online quantization configuration based on user patterns (#51285)
This commit is contained in:
@@ -67,7 +67,7 @@ quantization_config:
|
||||
moe:
|
||||
weight: <name>
|
||||
activation: <name>
|
||||
ignore: [<layer-name-or-regex>, ...]
|
||||
ignore: [<layer-name-or-regex-or-fnmatch-pattern>, ...]
|
||||
```
|
||||
|
||||
`linear` and `moe` accept a full `{weight, activation}` dict, or a bare
|
||||
@@ -133,7 +133,7 @@ llm = LLM(
|
||||
|
||||
### Excluding Layers from Quantization
|
||||
|
||||
Use the `ignore` parameter to skip specific layers. It accepts exact layer names and regex patterns (prefixed with `re:`):
|
||||
Use the `ignore` parameter to skip specific layers. It accepts exact layer names, regex patterns (prefixed with `re:`), and patterns understood by [`fnmatch.fnmatch`](https://docs.python.org/3/library/fnmatch.html#fnmatch.fnmatch):
|
||||
|
||||
```python
|
||||
from vllm import LLM
|
||||
@@ -147,10 +147,54 @@ llm = LLM(
|
||||
"model.layers.1.self_attn.o_proj",
|
||||
# regex: skip all QKV projections
|
||||
"re:.*[qkv]_proj",
|
||||
# fnmatch: skip all MoE experts
|
||||
"*mlp.experts*",
|
||||
],
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
!!! note
|
||||
For fused layers (e.g., `qkv_proj` which fuses `q_proj`, `k_proj`, `v_proj`), the ignore pattern must match the **unfused** shard names (`q_proj`, `k_proj`, `v_proj`), not the fused name.
|
||||
For fused layers (e.g., `qkv_proj` which fuses `q_proj`, `k_proj`, `v_proj`), patterns may match the fused name directly or all of its unfused shard names.
|
||||
|
||||
### Fine-Grained Per-Layer Quantization Schemes
|
||||
|
||||
Use the `targets` parameter to apply different online shorthands to different layers, instead of one scheme applied everywhere via `linear`/`moe`. Keys are exact layer names, regex patterns (prefixed with `re:`), or patterns understood by [`fnmatch.fnmatch`](https://docs.python.org/3/library/fnmatch.html#fnmatch.fnmatch); values are shorthand names (`fp8_per_tensor`, `fp8_per_block`, `fp8_per_channel`, `mxfp8`, `int8_per_channel_weight_only`, `nvfp4_per_token`).
|
||||
|
||||
Example:
|
||||
|
||||
```python
|
||||
from vllm import LLM
|
||||
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3.5-35B-A3B",
|
||||
quantization="online",
|
||||
quantization_config={
|
||||
"targets": {
|
||||
# exact layer name
|
||||
"model.layers.0.self_attn.o_proj": "fp8_per_tensor",
|
||||
# regex: quantize all QKV projections
|
||||
r"re:.*self_attn\.qkv_proj.*": "mxfp8",
|
||||
# fnmatch: quantize all MoE experts
|
||||
"*mlp.experts*": "mxfp4",
|
||||
},
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
Or with the CLI:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3.5-35B-A3B \
|
||||
--quantization online \
|
||||
--quantization-config '{"targets":{"model.layers.0.self_attn.o_proj":"fp8_per_tensor","re:.*self_attn\\.qkv_proj.*":"mxfp8","*mlp.experts*":"mxfp4"}}'
|
||||
```
|
||||
|
||||
!!! info
|
||||
|
||||
- `targets` is mutually exclusive with online `linear` and `moe`: set one or the other, not both.
|
||||
- A layer that matches no `targets` pattern is left unchanged from its checkpoint dtype.
|
||||
- A layer name may not match both `targets` and `ignore`, raising an error.
|
||||
- A layer may not match more than one `targets` pattern, raising an error.
|
||||
- fnmatch-style patterns are supported only by online quantization; they are
|
||||
not applied to Quark or compressed-tensors configurations.
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests quantization configuration matching utilities."""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
|
||||
should_ignore_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
_find_matching_targets,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
should_ignore_layer as quark_should_ignore_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
find_matching_patterns,
|
||||
get_layer_name_after_index,
|
||||
is_equal_or_regex_match,
|
||||
)
|
||||
|
||||
|
||||
def test_is_equal_or_regex_match():
|
||||
assert is_equal_or_regex_match(
|
||||
"model.layers.0.mlp.down_proj", "model.layers.0.mlp.down_proj"
|
||||
)
|
||||
assert is_equal_or_regex_match("model.layers.0.mlp.down_proj", r"re:.*down_proj")
|
||||
assert not is_equal_or_regex_match("model.layers.0.mlp.down_proj", "other")
|
||||
|
||||
|
||||
def test_fnmatch_is_opt_in():
|
||||
layer_name = "model.layers.0.mlp.experts"
|
||||
pattern = "*mlp.experts*"
|
||||
|
||||
assert not is_equal_or_regex_match(layer_name, pattern)
|
||||
assert is_equal_or_regex_match(layer_name, pattern, use_fnmatch=True)
|
||||
assert not should_ignore_layer(layer_name, [pattern])
|
||||
assert not quark_should_ignore_layer(layer_name, [pattern])
|
||||
assert should_ignore_layer(layer_name, [pattern], use_fnmatch=True)
|
||||
|
||||
|
||||
def test_online_targets_support_fnmatch_patterns():
|
||||
layer_name = "model.layers.0.mlp.experts"
|
||||
targets = {"*mlp.experts*": "mxfp4"}
|
||||
|
||||
assert _find_matching_targets(layer_name, targets) == ["*mlp.experts*"]
|
||||
|
||||
|
||||
def test_online_ignore_supports_fnmatch_patterns():
|
||||
layer_name = "model.layers.0.mlp.experts"
|
||||
ignore = ["*mlp.experts*"]
|
||||
config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(targets={"*mlp.experts*": "mxfp4"}, ignore=ignore)
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="matches both quantization_config.ignore"):
|
||||
config._dispatch_target(layer_name, Mock())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer_name,expected",
|
||||
[
|
||||
("model.layers.1.self_attn.qkv_proj", "self_attn.qkv_proj"),
|
||||
("model.layers.2.mlp.down_proj", "mlp.down_proj"),
|
||||
("lm_head", "lm_head"),
|
||||
],
|
||||
)
|
||||
def test_get_layer_name_after_index(layer_name, expected):
|
||||
assert get_layer_name_after_index(layer_name) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"patterns",
|
||||
[
|
||||
[r"re:.*qkv_proj.*"],
|
||||
[r"re:.*\.[qkv]_proj$"],
|
||||
],
|
||||
ids=["direct_fused_regex", "fused_shard_regexes"],
|
||||
)
|
||||
def test_find_matching_patterns_for_fused_regexes(patterns):
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
matches = find_matching_patterns(layer_name, patterns, fused_mapping)
|
||||
assert all(len(shard_matches) == 1 for shard_matches in matches)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"patterns,expected",
|
||||
[
|
||||
(
|
||||
[r"re:.*\.qkv_proj$"],
|
||||
[{r"re:.*\.qkv_proj$"}],
|
||||
),
|
||||
(
|
||||
[r"re:.*\.q_proj$", r"re:.*\.k_proj$", r"re:.*\.v_proj$"],
|
||||
[
|
||||
{r"re:.*\.q_proj$"},
|
||||
{r"re:.*\.k_proj$"},
|
||||
{r"re:.*\.v_proj$"},
|
||||
],
|
||||
),
|
||||
],
|
||||
ids=["direct_fused_match", "individual_shard_matches"],
|
||||
)
|
||||
def test_find_matching_patterns_distinguishes_direct_and_shard_matches(
|
||||
patterns, expected
|
||||
):
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
assert find_matching_patterns(layer_name, patterns, fused_mapping) == expected
|
||||
assert should_ignore_layer(layer_name, patterns, fused_mapping)
|
||||
|
||||
|
||||
def test_should_ignore_layer_rejects_partially_matched_fused_layer():
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
patterns = [r"re:.*\.q_proj$"]
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
assert find_matching_patterns(layer_name, patterns, fused_mapping) == [
|
||||
{r"re:.*\.q_proj$"},
|
||||
set(),
|
||||
set(),
|
||||
]
|
||||
with pytest.raises(ValueError, match="different quantization schemes"):
|
||||
should_ignore_layer(layer_name, patterns, fused_mapping)
|
||||
|
||||
|
||||
def test_should_ignore_layer_returns_false_when_no_fused_pattern_matches():
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
assert find_matching_patterns(layer_name, [], fused_mapping) == [
|
||||
set(),
|
||||
set(),
|
||||
set(),
|
||||
]
|
||||
assert not should_ignore_layer(layer_name, [], fused_mapping)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"patterns",
|
||||
[
|
||||
[r"re:.*qkv_proj.*"],
|
||||
[r"re:.*\.[qkv]_proj$"],
|
||||
],
|
||||
ids=["direct_fused_regex", "fused_shard_regexes"],
|
||||
)
|
||||
def test_ignore_and_targets_match_fused_regexes_identically(patterns):
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
targets = {pattern: "fp8_per_block" for pattern in patterns}
|
||||
|
||||
assert should_ignore_layer(layer_name, patterns, fused_mapping)
|
||||
matches = _find_matching_targets(layer_name, targets, fused_mapping)
|
||||
assert len(matches) == 1
|
||||
assert targets[matches[0]] == "fp8_per_block"
|
||||
|
||||
|
||||
def test_ignore_allows_individually_matched_fused_shards():
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
patterns = [
|
||||
"model.layers.0.self_attn.q_proj",
|
||||
"model.layers.0.self_attn.k_proj",
|
||||
"model.layers.0.self_attn.v_proj",
|
||||
]
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
assert should_ignore_layer(layer_name, patterns, fused_mapping)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"patterns",
|
||||
[
|
||||
["model.layers.0.self_attn.qkv_proj"],
|
||||
[r"re:.*\.qkv_proj$"],
|
||||
[
|
||||
"model.layers.0.self_attn.q_proj",
|
||||
"model.layers.0.self_attn.k_proj",
|
||||
"model.layers.0.self_attn.v_proj",
|
||||
],
|
||||
],
|
||||
ids=["direct_fused_name", "direct_fused_regex", "individual_shards"],
|
||||
)
|
||||
def test_quark_and_compressed_tensors_ignore_fused_layers_identically(patterns):
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
assert should_ignore_layer(layer_name, patterns, fused_mapping)
|
||||
assert quark_should_ignore_layer(layer_name, patterns, fused_mapping)
|
||||
@@ -18,10 +18,21 @@ from tests.quantization.utils import (
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm._custom_ops import scaled_fp4_quant
|
||||
from vllm.config.quantization import resolve_quantization_config
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.quantization import (
|
||||
QuantizationConfigArgs,
|
||||
resolve_quantization_config,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerBlockOnlineLinearMethod,
|
||||
Fp8PerBlockOnlineMoEMethod,
|
||||
@@ -49,6 +60,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
amax_for_tp_weight_quant,
|
||||
weight_amax,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.base_loader import log_online_quantization
|
||||
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
|
||||
from vllm.model_executor.models.granitemoe import (
|
||||
GraniteMoeModel,
|
||||
@@ -154,12 +169,23 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
Fp8PerBlockOnlineLinearMethod,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
),
|
||||
# quantization='online' with per-layer target patterns
|
||||
(
|
||||
"online",
|
||||
{
|
||||
"targets": {
|
||||
r"re:.*self_attn\.o_proj": "fp8_per_block",
|
||||
r"re:.*block_sparse_moe\.experts": "fp8_per_tensor",
|
||||
}
|
||||
},
|
||||
Fp8PerBlockOnlineLinearMethod,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
),
|
||||
# ignore with direct layer name
|
||||
(
|
||||
"fp8_per_tensor",
|
||||
# qkv_proj is fused from q_proj/k_proj/v_proj, so currently the
|
||||
# ignore regex must match the unfused shard names
|
||||
# TODO(future PR): also make 're:.*qkv_proj.*' work
|
||||
# qkv_proj is fused from q_proj/k_proj/v_proj. The shard regex
|
||||
# remains supported alongside direct fused-name regexes.
|
||||
{"ignore": ["model.layers.1.self_attn.o_proj", "re:.*[qkv]_proj"]},
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
@@ -171,6 +197,15 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
Mxfp4OnlineMoEMethod,
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"fp8_per_tensor",
|
||||
"fp8_per_block",
|
||||
"fp8_per_channel",
|
||||
"per_layer_kind_overrides",
|
||||
"targets",
|
||||
"ignore",
|
||||
"mxfp4",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
|
||||
@@ -266,6 +301,12 @@ def test_online_quantization(
|
||||
layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod
|
||||
)
|
||||
|
||||
if isinstance(online_quant_args, dict) and "targets" in online_quant_args:
|
||||
for layer in model.model.layers:
|
||||
assert isinstance(
|
||||
layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod
|
||||
)
|
||||
|
||||
input_ids = torch.tensor([1, 2, 3, 4], device=DEVICE)
|
||||
positions = torch.arange(input_ids.numel(), device=DEVICE)
|
||||
with set_forward_context(None, vllm_config, num_tokens=input_ids.numel()):
|
||||
@@ -321,6 +362,158 @@ def test_online_quantization_loads_real_weights(vllm_runner, monkeypatch) -> Non
|
||||
assert outputs
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"targets,prefix,expected_method_cls,unmatched_prefix,expected_metadata",
|
||||
[
|
||||
(
|
||||
{r"re:.*self_attn\.o_proj": "fp8_per_block"},
|
||||
"model.layers.0.self_attn.o_proj",
|
||||
Fp8PerBlockOnlineLinearMethod,
|
||||
"model.layers.0.self_attn.qkv_proj",
|
||||
("targets", "fp8_per_block", r"re:.*self_attn\.o_proj"),
|
||||
),
|
||||
(
|
||||
{r"re:.*qkv_proj.*": "fp8_per_tensor"},
|
||||
"model.layers.0.self_attn.qkv_proj",
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
"model.layers.0.self_attn.o_proj",
|
||||
("targets", "fp8_per_tensor", r"re:.*qkv_proj.*"),
|
||||
),
|
||||
(
|
||||
{r"re:.*[qkv]_proj": "fp8_per_tensor"},
|
||||
"model.layers.0.self_attn.qkv_proj",
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
"model.layers.0.self_attn.o_proj",
|
||||
("targets", "fp8_per_tensor", r"re:.*[qkv]_proj"),
|
||||
),
|
||||
],
|
||||
ids=["linear_regex", "direct_fused_regex", "legacy_fused_regex"],
|
||||
)
|
||||
def test_online_quantization_targets(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
targets: dict[str, str],
|
||||
prefix: str,
|
||||
expected_method_cls,
|
||||
unmatched_prefix: str,
|
||||
expected_metadata: tuple[str, str, str],
|
||||
) -> None:
|
||||
"""Target patterns select the real online linear methods."""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
config = OnlineQuantizationConfig(QuantizationConfigArgs(targets=targets))
|
||||
config.packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
layer = ColumnParallelLinear(
|
||||
input_size=1,
|
||||
output_size=1,
|
||||
bias=False,
|
||||
disable_tp=True,
|
||||
)
|
||||
|
||||
method = config.get_quant_method(layer, prefix)
|
||||
assert isinstance(method, expected_method_cls)
|
||||
assert config.quantized_layers == {prefix: expected_metadata}
|
||||
|
||||
unmatched_method = config.get_quant_method(layer, unmatched_prefix)
|
||||
assert isinstance(unmatched_method, UnquantizedLinearMethod)
|
||||
assert config.quantized_layers == {prefix: expected_metadata}
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_online_quantization_records_global_config(
|
||||
default_vllm_config, dist_init
|
||||
) -> None:
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_block"))
|
||||
prefix = "model.layers.0.self_attn.o_proj"
|
||||
layer = ColumnParallelLinear(
|
||||
input_size=1,
|
||||
output_size=1,
|
||||
bias=False,
|
||||
disable_tp=True,
|
||||
)
|
||||
|
||||
method = config.get_quant_method(layer, prefix)
|
||||
|
||||
assert isinstance(method, Fp8PerBlockOnlineLinearMethod)
|
||||
assert config.quantized_layers == {
|
||||
prefix: ("linear", str(config.args.linear), None)
|
||||
}
|
||||
|
||||
|
||||
def test_online_quantization_targets_ignore_collision() -> None:
|
||||
"""A targets/ignore collision is reported when the layer is dispatched."""
|
||||
config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(
|
||||
targets={"model.layers.0.self_attn.o_proj": "fp8_per_tensor"},
|
||||
ignore=["model.layers.0.self_attn.o_proj"],
|
||||
)
|
||||
)
|
||||
with pytest.raises(ValueError, match="matches both quantization_config.ignore"):
|
||||
config._dispatch_target(
|
||||
"model.layers.0.self_attn.o_proj", Mock(spec=LinearBase)
|
||||
)
|
||||
|
||||
|
||||
def test_online_quantization_targets_reject_unsupported_layer() -> None:
|
||||
"""A targets match on a non-linear, non-MoE layer is rejected."""
|
||||
config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(targets={"lm_head": "fp8_per_tensor"})
|
||||
)
|
||||
lm_head = VocabParallelEmbedding(
|
||||
num_embeddings=1,
|
||||
embedding_dim=1,
|
||||
disable_tp=True,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"Layer lm_head was matched by quantization_config.targets "
|
||||
r"\(lm_head\), but online quantization is not supported for "
|
||||
"VocabParallelEmbedding."
|
||||
),
|
||||
):
|
||||
config.get_quant_method(lm_head, "lm_head")
|
||||
|
||||
|
||||
def test_log_online_quantization(default_vllm_config, monkeypatch) -> None:
|
||||
config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_tensor"))
|
||||
config.quantized_layers = {
|
||||
"model.layers.0.mlp.down_proj": ("linear", "fp8_per_tensor", None),
|
||||
"model.layers.1.mlp.down_proj": ("linear", "fp8_per_tensor", None),
|
||||
"model.layers.0.self_attn.qkv_proj": (
|
||||
"targets",
|
||||
"mxfp4",
|
||||
r"re:.*qkv_proj.*",
|
||||
),
|
||||
}
|
||||
default_vllm_config.quant_config = config
|
||||
|
||||
logged_messages: list[str] = []
|
||||
|
||||
def record_info(message: str, *args: object) -> None:
|
||||
logged_messages.append(message % args)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.base_loader.logger.info", record_info
|
||||
)
|
||||
log_online_quantization(default_vllm_config)
|
||||
|
||||
assert logged_messages == [
|
||||
"Quantized 3 layers of types: mlp.down_proj: 2 (from linear: "
|
||||
"fp8_per_tensor); self_attn.qkv_proj: 1 (from targets: "
|
||||
"re:.*qkv_proj.*, mxfp4)"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_cuda()
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for QuantizationConfigArgs parsing."""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.quantization import (
|
||||
@@ -10,6 +12,11 @@ from vllm.config.quantization import (
|
||||
QuantSpec,
|
||||
resolve_quantization_config,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
_find_matching_targets,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTokenSym,
|
||||
@@ -34,6 +41,10 @@ def test_quant_spec_accepts_quant_key_directly():
|
||||
assert spec.activation is None
|
||||
|
||||
|
||||
def test_quant_spec_string_representation_uses_quantization_name():
|
||||
assert str(QuantSpec(weight="mxfp4")) == "mxfp4"
|
||||
|
||||
|
||||
def test_quant_spec_rejects_unknown_name():
|
||||
with pytest.raises(ValueError, match="unknown quantization name"):
|
||||
QuantSpec(weight="not_a_real_format")
|
||||
@@ -70,6 +81,11 @@ def test_args_accepts_dict_form():
|
||||
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
|
||||
|
||||
|
||||
def test_targets_reject_non_string_keys():
|
||||
with pytest.raises(ValueError, match="targets keys must be strings"):
|
||||
QuantizationConfigArgs._validate_targets({123: "mxfp8"})
|
||||
|
||||
|
||||
# ---- resolve_quantization_config -----------------------------------------
|
||||
|
||||
|
||||
@@ -132,3 +148,64 @@ def test_static_block_weight_paired_with_dynamic_block_activation():
|
||||
spec = QuantSpec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
|
||||
assert spec.weight == kFp8Static128BlockSym
|
||||
assert spec.activation == kFp8Dynamic128Sym
|
||||
|
||||
|
||||
def test_targets_allow_distinct_patterns_with_the_same_shorthand():
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
targets = {
|
||||
r"re:.*\.q_proj$": "mxfp8",
|
||||
r"re:.*\.k_proj$": "mxfp8",
|
||||
r"re:.*\.v_proj$": "mxfp8",
|
||||
}
|
||||
|
||||
matches = _find_matching_targets(layer_name, targets, fused_mapping)
|
||||
assert len(matches) == 1
|
||||
assert targets[matches[0]] == "mxfp8"
|
||||
|
||||
|
||||
def test_targets_reject_overlapping_patterns():
|
||||
targets = {
|
||||
r"re:.*o_proj": "fp8_per_tensor",
|
||||
"model.layers.0.self_attn.o_proj": "fp8_per_block",
|
||||
}
|
||||
|
||||
with pytest.raises(ValueError, match="multiple quantization_config.targets"):
|
||||
_find_matching_targets("model.layers.0.self_attn.o_proj", targets)
|
||||
|
||||
|
||||
def test_targets_reject_partially_matched_fused_layer():
|
||||
targets = {r"re:.*q_proj": "fp8_per_tensor"}
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
with pytest.raises(ValueError, match="unmatched shards"):
|
||||
_find_matching_targets(
|
||||
"model.layers.0.self_attn.qkv_proj", targets, fused_mapping
|
||||
)
|
||||
|
||||
|
||||
def test_targets_reject_fused_shards_with_different_schemes():
|
||||
targets = {
|
||||
r"re:.*\.q_proj$": "fp8_per_tensor",
|
||||
r"re:.*\.k_proj$": "fp8_per_block",
|
||||
r"re:.*\.v_proj$": "fp8_per_tensor",
|
||||
}
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
with pytest.raises(ValueError, match="different quantization_config.targets"):
|
||||
_find_matching_targets(
|
||||
"model.layers.0.self_attn.qkv_proj", targets, fused_mapping
|
||||
)
|
||||
|
||||
|
||||
def test_targets_reject_moe_only_shorthand_for_linear_layer():
|
||||
config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(
|
||||
targets={"model.layers.0.self_attn.o_proj": "nvfp4_per_token"}
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not define a QuantSpec"):
|
||||
config._dispatch_target(
|
||||
"model.layers.0.self_attn.o_proj", Mock(spec=LinearBase)
|
||||
)
|
||||
|
||||
@@ -50,7 +50,10 @@ from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E
|
||||
QuarkW8A8Int8MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
|
||||
from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
QuarkQTensorHint,
|
||||
should_ignore_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
@@ -1294,6 +1297,23 @@ FUSED_MAPPING = {
|
||||
}
|
||||
|
||||
|
||||
def test_quark_should_ignore_layer_checks_children():
|
||||
assert should_ignore_layer(
|
||||
"model.layers.78.mlp.experts",
|
||||
["model.layers.78.mlp.experts.0.down_proj"],
|
||||
check_children=True,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_should_ignore_layer_rejects_partial_fused_matches():
|
||||
with pytest.raises(ValueError, match="different quantization schemes"):
|
||||
should_ignore_layer(
|
||||
"model.layers.0.self_attn.qkv_proj",
|
||||
["model.layers.0.self_attn.q_proj"],
|
||||
FUSED_MAPPING,
|
||||
)
|
||||
|
||||
|
||||
def test_fused_name_listed_directly_is_skipped():
|
||||
# Regression for Step-3.5-Flash-FP8: the checkpoint lists the fused
|
||||
# name (``qkv_proj``) directly in ``modules_to_not_convert``. When a
|
||||
|
||||
@@ -3,7 +3,14 @@
|
||||
|
||||
from typing import Annotated, Any
|
||||
|
||||
from pydantic import Field, GetPydanticSchema, ValidationInfo, field_validator
|
||||
import regex as re
|
||||
from pydantic import (
|
||||
Field,
|
||||
GetPydanticSchema,
|
||||
ValidationInfo,
|
||||
field_validator,
|
||||
model_validator,
|
||||
)
|
||||
from pydantic_core import core_schema
|
||||
|
||||
from vllm.config.utils import config
|
||||
@@ -76,6 +83,21 @@ class QuantSpec:
|
||||
activation: QuantKeyField = None
|
||||
"""Activation quantization key, or a name from QUANT_KEY_NAMES."""
|
||||
|
||||
def __str__(self) -> str:
|
||||
def quant_key_str(quant_key: QuantKey | None) -> str:
|
||||
if quant_key is None:
|
||||
return "None"
|
||||
return next(
|
||||
(
|
||||
name
|
||||
for name, known_quant_key in QUANT_KEY_NAMES.items()
|
||||
if known_quant_key == quant_key
|
||||
),
|
||||
str(quant_key),
|
||||
)
|
||||
|
||||
return quant_key_str(self.weight)
|
||||
|
||||
|
||||
@config
|
||||
class QuantizationConfigArgs:
|
||||
@@ -92,7 +114,16 @@ class QuantizationConfigArgs:
|
||||
"""Spec applied to ``FusedMoEFactory`` layers."""
|
||||
|
||||
ignore: list[str] = Field(default_factory=list)
|
||||
"""Layers to skip quantization for."""
|
||||
"""Layers to skip quantization for. Online quantization also supports
|
||||
fnmatch-style patterns."""
|
||||
|
||||
targets: dict[str, str] | None = None
|
||||
"""Per-layer online quantization overrides, keyed by exact layer name or
|
||||
regex patterns with a `re:`, or fnmatch-style patterns for online
|
||||
quantization, mapping to an online shorthand name (see
|
||||
`_ONLINE_SHORTHANDS`). A layer that matches no pattern is left unquantized.
|
||||
Mutually exclusive with `linear` and `moe`.
|
||||
"""
|
||||
|
||||
@field_validator("linear", "moe", mode="before")
|
||||
@classmethod
|
||||
@@ -110,6 +141,46 @@ class QuantizationConfigArgs:
|
||||
return spec
|
||||
return QuantSpec(weight=_coerce_quant_key(v))
|
||||
|
||||
@field_validator("targets", mode="before")
|
||||
@classmethod
|
||||
def _validate_targets(cls, v: Any) -> Any:
|
||||
if v is None:
|
||||
return v
|
||||
if not isinstance(v, dict):
|
||||
raise TypeError(f"targets must be a dict, got {type(v).__name__}")
|
||||
for pattern, shorthand in v.items():
|
||||
if not isinstance(pattern, str):
|
||||
raise ValueError(
|
||||
f"targets keys must be strings, got {type(pattern).__name__}"
|
||||
)
|
||||
if not isinstance(shorthand, str) or shorthand not in _ONLINE_SHORTHANDS:
|
||||
raise ValueError(
|
||||
f"targets[{pattern}] = {shorthand} is not a valid "
|
||||
f"online shorthand name; expected one of "
|
||||
f"{sorted(_ONLINE_SHORTHANDS)}"
|
||||
)
|
||||
if pattern.startswith("re:"):
|
||||
try:
|
||||
re.compile(pattern[3:])
|
||||
except re.error as e:
|
||||
raise ValueError(
|
||||
f"targets key {pattern} is not a valid regex: {e}"
|
||||
) from e
|
||||
return v
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_targets_exclusivity(self) -> "QuantizationConfigArgs":
|
||||
if self.targets is None:
|
||||
return self
|
||||
if self.linear is not None or self.moe is not None:
|
||||
raise ValueError(
|
||||
"quantization_config.targets is mutually exclusive with "
|
||||
f"quantization_config.linear/moe, got "
|
||||
f"targets={self.targets}, linear={self.linear}, "
|
||||
f"moe={self.moe}."
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
# CLI shorthands accepted by `--quantization`. Each desugars to a full
|
||||
# QuantizationConfigArgs; activation overrides go through quantization_config.
|
||||
@@ -191,4 +262,5 @@ def resolve_quantization_config(
|
||||
linear=quantization_config.linear or base.linear,
|
||||
moe=quantization_config.moe or base.moe,
|
||||
ignore=quantization_config.ignore or base.ignore,
|
||||
targets=quantization_config.targets or base.targets,
|
||||
)
|
||||
|
||||
@@ -4,11 +4,14 @@
|
||||
from collections.abc import Iterable, Mapping
|
||||
from types import MappingProxyType
|
||||
|
||||
import regex as re
|
||||
from compressed_tensors import CompressionFormat
|
||||
from compressed_tensors.quantization import QuantizationStrategy
|
||||
from torch.nn import Module
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
find_matching_patterns,
|
||||
is_equal_or_regex_match,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
@@ -51,63 +54,20 @@ def should_ignore_layer(
|
||||
layer_name: str | None,
|
||||
ignore: Iterable[str] = tuple(),
|
||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
|
||||
use_fnmatch: bool = False,
|
||||
) -> bool:
|
||||
if layer_name is None:
|
||||
return False
|
||||
|
||||
# layer_name = model.layers.0.self_attn.qkv_proj
|
||||
# proj_name = qkv_proj
|
||||
proj_name = layer_name.split(".")[-1]
|
||||
|
||||
# Fused layers like gate_up_proj or qkv_proj will not be fused
|
||||
# in the safetensors checkpoint. So, we convert the name
|
||||
# from the fused version to unfused + check to make sure that
|
||||
# each shard of the fused layer has the same scheme.
|
||||
if proj_name in fused_mapping and layer_name not in ignore:
|
||||
shard_proj_names = fused_mapping[proj_name]
|
||||
|
||||
# Convert fused_name --> [shard_names]
|
||||
shard_names = [
|
||||
layer_name.replace(proj_name, shard_proj_name)
|
||||
for shard_proj_name in shard_proj_names
|
||||
]
|
||||
|
||||
# Layer should be ignored if shards are ignored.
|
||||
should_ignore_layer = None
|
||||
for shard_name in shard_names:
|
||||
should_ignore_shard = check_equal_or_regex_match(
|
||||
layer_name=shard_name, targets=ignore
|
||||
)
|
||||
|
||||
# If shard_idx=0, set layer ignore to match shard.
|
||||
if should_ignore_layer is None:
|
||||
should_ignore_layer = should_ignore_shard
|
||||
|
||||
# If shard_idx=1+ confirm scheme matches prior shards.
|
||||
elif should_ignore_shard != should_ignore_layer:
|
||||
raise ValueError(
|
||||
f"Found a different quantization schemes for "
|
||||
f"{shard_proj_names} in {layer_name}. vLLM "
|
||||
"requires all to use the same scheme."
|
||||
)
|
||||
|
||||
# Unfused layers like down_proj and o_proj will match
|
||||
# the safetensors checkpoint already.
|
||||
else:
|
||||
should_ignore_layer = check_equal_or_regex_match(
|
||||
layer_name=layer_name, targets=ignore
|
||||
per_shard_matches = find_matching_patterns(
|
||||
layer_name, ignore, fused_mapping, use_fnmatch=use_fnmatch
|
||||
)
|
||||
shards_ignored = [len(matches) > 0 for matches in per_shard_matches]
|
||||
if any(shards_ignored) and not all(shards_ignored):
|
||||
raise ValueError(
|
||||
f"Found different quantization schemes for the shards of "
|
||||
f"{layer_name}. vLLM requires all to use the same scheme."
|
||||
)
|
||||
|
||||
assert should_ignore_layer is not None
|
||||
return should_ignore_layer
|
||||
|
||||
|
||||
def check_equal_or_regex_match(layer_name: str, targets: Iterable[str]) -> bool:
|
||||
"""
|
||||
Checks whether a layer_name is exactly equal or a regex match for
|
||||
if target starts with 're:' to any target in list.
|
||||
"""
|
||||
return any(_is_equal_or_regex_match(layer_name, target) for target in targets)
|
||||
return all(shards_ignored)
|
||||
|
||||
|
||||
def find_matched_target(
|
||||
@@ -167,32 +127,11 @@ def _find_first_match(
|
||||
"""
|
||||
|
||||
for target in targets:
|
||||
if _is_equal_or_regex_match(value, target, check_contains=check_contains):
|
||||
if is_equal_or_regex_match(value, target, check_contains=check_contains):
|
||||
return target
|
||||
return None
|
||||
|
||||
|
||||
def _is_equal_or_regex_match(
|
||||
value: str, target: str, check_contains: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Checks whether a value is exactly equal or a regex match for target
|
||||
if target starts with 're:'. If check_contains is set to True,
|
||||
additionally checks if the target string is contained within the value.
|
||||
"""
|
||||
|
||||
if target.startswith("re:"):
|
||||
pattern = target[3:]
|
||||
if re.match(pattern, value):
|
||||
return True
|
||||
elif check_contains:
|
||||
if target.lower() in value.lower():
|
||||
return True
|
||||
elif target == value:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _match_fused_layer(
|
||||
layer_name: str,
|
||||
target_layers: Iterable[str],
|
||||
@@ -230,7 +169,7 @@ def _match_fused_layer(
|
||||
unfused_matches: list[str | None] = []
|
||||
for unfused in unfused_paths:
|
||||
for target in target_layers:
|
||||
if _is_equal_or_regex_match(unfused, target):
|
||||
if is_equal_or_regex_match(unfused, target):
|
||||
unfused_matches.append(target)
|
||||
break
|
||||
else:
|
||||
|
||||
@@ -1,11 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
|
||||
from vllm.config.quantization import (
|
||||
_ONLINE_SHORTHANDS,
|
||||
QuantizationConfigArgs,
|
||||
QuantSpec,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
@@ -47,6 +53,10 @@ from vllm.model_executor.layers.quantization.online.mxfp8 import (
|
||||
from vllm.model_executor.layers.quantization.online.nvfp4 import (
|
||||
Nvfp4OnlineMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
find_matching_patterns,
|
||||
get_layer_name_after_index,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Static128BlockSym,
|
||||
@@ -83,6 +93,39 @@ _ONLINE_MOE_METHODS: dict[QuantKey, type] = {
|
||||
}
|
||||
|
||||
|
||||
def _find_matching_targets(
|
||||
prefix: str,
|
||||
targets: Mapping[str, str],
|
||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
|
||||
) -> list[str]:
|
||||
per_shard_matches = find_matching_patterns(
|
||||
prefix, targets, fused_mapping, use_fnmatch=True
|
||||
)
|
||||
if all(len(matches) == 0 for matches in per_shard_matches):
|
||||
return []
|
||||
if any(len(matches) == 0 for matches in per_shard_matches):
|
||||
raise ValueError(
|
||||
f"Found unmatched shards for {prefix}: {per_shard_matches}. vLLM "
|
||||
"requires all shards of a fused layer to match a target."
|
||||
)
|
||||
if any(len(matches) > 1 for matches in per_shard_matches):
|
||||
raise ValueError(
|
||||
f"Found multiple quantization_config.targets matches for the "
|
||||
f"shards of {prefix}: {per_shard_matches}. Each shard may match "
|
||||
"at most one target."
|
||||
)
|
||||
|
||||
matched_patterns = [next(iter(matches)) for matches in per_shard_matches]
|
||||
quant_key_strs = {targets[pattern] for pattern in matched_patterns}
|
||||
if len(quant_key_strs) > 1:
|
||||
raise ValueError(
|
||||
f"Found different quantization_config.targets values for the "
|
||||
f"shards of {prefix}: {matched_patterns}. vLLM requires all "
|
||||
"shards of a fused layer to use the same target."
|
||||
)
|
||||
return [matched_patterns[0]]
|
||||
|
||||
|
||||
class OnlineQuantizationConfig(QuantizationConfig):
|
||||
"""Model-level config for online quantization (quantize fp16/bf16 weights
|
||||
during model loading, without requiring a pre-quantized checkpoint)."""
|
||||
@@ -92,14 +135,44 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
args: QuantizationConfigArgs,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if args.linear is None and args.moe is None:
|
||||
if args.linear is None and args.moe is None and args.targets is None:
|
||||
raise ValueError(
|
||||
"OnlineQuantizationConfig requires at least one of "
|
||||
"quantization_config.linear or quantization_config.moe "
|
||||
"to be set."
|
||||
"quantization_config.linear, quantization_config.moe, or "
|
||||
"quantization_config.targets to be set."
|
||||
)
|
||||
self.args = args
|
||||
self.ignored_layers: list[str] = args.ignore
|
||||
self.quantized_layers: dict[str, tuple[str, str, str | None]] = {}
|
||||
|
||||
@property
|
||||
def quantized_layer_summaries(self) -> list[str]:
|
||||
counts: dict[tuple[str, str, str | None, str], int] = {}
|
||||
for layer_name, (
|
||||
source,
|
||||
quant_key_str,
|
||||
target_pattern,
|
||||
) in self.quantized_layers.items():
|
||||
key = (
|
||||
get_layer_name_after_index(layer_name),
|
||||
source,
|
||||
target_pattern,
|
||||
quant_key_str,
|
||||
)
|
||||
counts[key] = counts.get(key, 0) + 1
|
||||
|
||||
summaries = []
|
||||
# Build summary entries as
|
||||
# `self_attn.o_proj: 24 (from targets: re:.*self_attn\.o_proj, mxfp4`
|
||||
for (layer_type, source, target_pattern, quant_key_str), count in sorted(
|
||||
counts.items()
|
||||
):
|
||||
pattern_prefix = f"{target_pattern}, " if target_pattern else ""
|
||||
summaries.append(
|
||||
f"{layer_type}: {count} "
|
||||
f"(from {source}: {pattern_prefix}{quant_key_str})"
|
||||
)
|
||||
return summaries
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
@@ -154,29 +227,101 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
return cls(layer=layer)
|
||||
return cls()
|
||||
|
||||
def _dispatch_target(
|
||||
self, prefix: str, layer: torch.nn.Module
|
||||
) -> "QuantizeMethodBase | None":
|
||||
assert self.args.targets is not None
|
||||
ignored = should_ignore_layer(
|
||||
prefix,
|
||||
ignore=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
use_fnmatch=True,
|
||||
)
|
||||
matches = _find_matching_targets(
|
||||
prefix, self.args.targets, fused_mapping=self.packed_modules_mapping
|
||||
)
|
||||
if ignored and matches:
|
||||
raise ValueError(
|
||||
f"Layer {prefix} matches both quantization_config.ignore "
|
||||
f"and quantization_config.targets ({matches}); a layer may "
|
||||
f"not be referenced by both."
|
||||
)
|
||||
if ignored or not matches:
|
||||
return None
|
||||
if len(matches) > 1:
|
||||
raise ValueError(
|
||||
f"Layer {prefix} matches multiple quantization_config."
|
||||
f"targets patterns: {matches}. Each layer may match at most "
|
||||
f"one target."
|
||||
)
|
||||
quant_key_str = self.args.targets[matches[0]]
|
||||
shorthand = _ONLINE_SHORTHANDS[quant_key_str]
|
||||
if isinstance(layer, LinearBase):
|
||||
quant_spec = shorthand.linear
|
||||
table = _ONLINE_LINEAR_METHODS
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
quant_spec = shorthand.moe
|
||||
table = _ONLINE_MOE_METHODS
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Layer {prefix} was matched by quantization_config.targets "
|
||||
f"({matches[0]}), but online quantization is not supported for "
|
||||
f"{type(layer).__name__}."
|
||||
)
|
||||
if quant_spec is None:
|
||||
raise ValueError(
|
||||
f"targets pattern {matches[0]} = {quant_key_str} does "
|
||||
f"not define a QuantSpec for {type(layer).__name__} layers "
|
||||
f"(matched at {prefix})."
|
||||
)
|
||||
method = self._dispatch(quant_spec, table, layer)
|
||||
if method is not None:
|
||||
self.quantized_layers[prefix] = ("targets", quant_key_str, matches[0])
|
||||
return method
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
# `targets` takes precedence over `moe` and `linear` and is exclusive.
|
||||
if self.args.targets is not None:
|
||||
method = self._dispatch_target(prefix, layer)
|
||||
if method is not None:
|
||||
return method
|
||||
|
||||
if isinstance(layer, LinearBase):
|
||||
return UnquantizedLinearMethod()
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
return None
|
||||
|
||||
if isinstance(layer, LinearBase):
|
||||
if should_ignore_layer(
|
||||
prefix,
|
||||
ignore=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
use_fnmatch=True,
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer)
|
||||
return method if method is not None else UnquantizedLinearMethod()
|
||||
if method is not None:
|
||||
self.quantized_layers[prefix] = (
|
||||
"linear",
|
||||
str(self.args.linear),
|
||||
None,
|
||||
)
|
||||
return method
|
||||
return UnquantizedLinearMethod()
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
if should_ignore_layer(
|
||||
prefix,
|
||||
ignore=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
use_fnmatch=True,
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer)
|
||||
return (
|
||||
method
|
||||
if method is not None
|
||||
else UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
)
|
||||
if method is not None:
|
||||
self.quantized_layers[prefix] = ("moe", str(self.args.moe), None)
|
||||
return method
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
return None
|
||||
|
||||
@@ -5,9 +5,12 @@ from collections.abc import Iterable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
find_matching_patterns,
|
||||
)
|
||||
|
||||
QuarkQTensorHint = dict[str, Any] | list[dict[str, Any]] | None
|
||||
|
||||
|
||||
@@ -49,80 +52,18 @@ def should_ignore_layer(
|
||||
):
|
||||
return True
|
||||
|
||||
# layer_name = model.layers.0.self_attn.qkv_proj
|
||||
# proj_name = qkv_proj
|
||||
proj_name = layer_name.split(".")[-1]
|
||||
|
||||
# Fused layers like gate_up_proj or qkv_proj will not be fused
|
||||
# in the safetensors checkpoint. So, we convert the name
|
||||
# from the fused version to unfused + check to make sure that
|
||||
# each shard of the fused layer has the same scheme.
|
||||
if proj_name in fused_mapping:
|
||||
shard_proj_names = fused_mapping[proj_name]
|
||||
|
||||
# Convert fused_name --> [shard_names]
|
||||
shard_names = [
|
||||
layer_name.replace(proj_name, shard_proj_name)
|
||||
for shard_proj_name in shard_proj_names
|
||||
]
|
||||
|
||||
# Layer should be ignored if shards are ignored.
|
||||
should_ignore_layer = None
|
||||
for shard_name in shard_names:
|
||||
should_ignore_shard = check_equal_or_regex_match(
|
||||
layer_name=shard_name, targets=ignore
|
||||
)
|
||||
|
||||
# If shard_idx=0, set layer ignore to match shard.
|
||||
if should_ignore_layer is None:
|
||||
should_ignore_layer = should_ignore_shard
|
||||
|
||||
# If shard_idx=1+ confirm scheme matches prior shards.
|
||||
elif should_ignore_shard != should_ignore_layer:
|
||||
raise ValueError(
|
||||
f"Found a different quantization schemes for "
|
||||
f"{shard_proj_names} in {layer_name}. vLLM "
|
||||
"requires all to use the same scheme."
|
||||
)
|
||||
|
||||
# Unfused layers like down_proj and o_proj will match
|
||||
# the safetensors checkpoint already.
|
||||
else:
|
||||
should_ignore_layer = check_equal_or_regex_match(
|
||||
layer_name=layer_name, targets=ignore
|
||||
# A direct fused-layer pattern takes precedence over expansion. For
|
||||
# model.layers.0.self_attn.qkv_proj,
|
||||
# ignore=["re:.*qkv_proj.*"] yields [{"re:.*qkv_proj.*"}]. In contrast,
|
||||
# ignore=["re:.*[qkv]_proj"] yields one matching set per expanded shard.
|
||||
per_shard_matches = find_matching_patterns(layer_name, ignore, fused_mapping)
|
||||
shards_ignored = [len(matches) > 0 for matches in per_shard_matches]
|
||||
if any(shards_ignored) and not all(shards_ignored):
|
||||
raise ValueError(
|
||||
f"Found different quantization schemes for the shards of "
|
||||
f"{layer_name}. vLLM requires all to use the same scheme."
|
||||
)
|
||||
|
||||
assert should_ignore_layer is not None
|
||||
return should_ignore_layer
|
||||
|
||||
|
||||
def check_equal_or_regex_match(layer_name: str, targets: Iterable[str]) -> bool:
|
||||
"""
|
||||
Checks whether a layer_name is exactly equal or a regex match for
|
||||
if target starts with 're:' to any target in list.
|
||||
"""
|
||||
return any(_is_equal_or_regex_match(layer_name, target) for target in targets)
|
||||
|
||||
|
||||
def _is_equal_or_regex_match(
|
||||
value: str, target: str, check_contains: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Checks whether a value is exactly equal or a regex match for target
|
||||
if target starts with 're:'. If check_contains is set to True,
|
||||
additionally checks if the target string is contained within the value.
|
||||
"""
|
||||
|
||||
if target.startswith("re:"):
|
||||
pattern = target[3:]
|
||||
if re.match(pattern, value):
|
||||
return True
|
||||
elif check_contains:
|
||||
if target.lower() in value.lower():
|
||||
return True
|
||||
elif target == value:
|
||||
return True
|
||||
return False
|
||||
return all(shards_ignored)
|
||||
|
||||
|
||||
# utility for tensor dims > 2 cases
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from fnmatch import fnmatch
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
@@ -244,3 +247,76 @@ def is_shared_expert_quant_fse_compatible(
|
||||
"shared-expert FSE quantization compatibility is not implemented for "
|
||||
f"{type(quant_config).__name__}",
|
||||
)
|
||||
|
||||
|
||||
def find_matching_patterns(
|
||||
layer_name: str,
|
||||
patterns: Iterable[str],
|
||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
|
||||
use_fnmatch: bool = False,
|
||||
) -> list[set[str]]:
|
||||
"""Return matching patterns for a layer or each shard of a fused layer.
|
||||
|
||||
A pattern matching the fused layer directly takes precedence. Otherwise,
|
||||
return one set of matching patterns for every shard.
|
||||
"""
|
||||
patterns = list(patterns)
|
||||
matches = [
|
||||
pattern
|
||||
for pattern in patterns
|
||||
if is_equal_or_regex_match(layer_name, pattern, use_fnmatch=use_fnmatch)
|
||||
]
|
||||
if matches:
|
||||
return [set(matches)]
|
||||
|
||||
proj_name = layer_name.split(".")[-1]
|
||||
if proj_name not in fused_mapping:
|
||||
return [set()]
|
||||
|
||||
shard_names = [
|
||||
layer_name.replace(proj_name, shard_proj_name)
|
||||
for shard_proj_name in fused_mapping[proj_name]
|
||||
]
|
||||
per_shard_matches = [
|
||||
{
|
||||
pattern
|
||||
for pattern in patterns
|
||||
if is_equal_or_regex_match(shard_name, pattern, use_fnmatch=use_fnmatch)
|
||||
}
|
||||
for shard_name in shard_names
|
||||
]
|
||||
return per_shard_matches
|
||||
|
||||
|
||||
def get_layer_name_after_index(layer_name: str) -> str:
|
||||
"""Return the suffix following the final numeric component of a layer name."""
|
||||
parts = layer_name.split(".")
|
||||
for index in range(len(parts) - 1, -1, -1):
|
||||
if parts[index].isdigit():
|
||||
return ".".join(parts[index + 1 :])
|
||||
return layer_name
|
||||
|
||||
|
||||
def is_equal_or_regex_match(
|
||||
value: str,
|
||||
target: str,
|
||||
check_contains: bool = False,
|
||||
use_fnmatch: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
Checks whether a value is exactly equal or a regex match for target
|
||||
if target starts with 're:'. If check_contains is set to True,
|
||||
additionally checks if the target string is contained within the value.
|
||||
If use_fnmatch is set, supports shell-style patterns in target.
|
||||
"""
|
||||
|
||||
if target.startswith("re:"):
|
||||
pattern = target[3:]
|
||||
if re.match(pattern, value):
|
||||
return True
|
||||
elif check_contains:
|
||||
if target.lower() in value.lower():
|
||||
return True
|
||||
elif target == value or use_fnmatch and fnmatch(value, target):
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -58,6 +58,7 @@ class BaseModelLoader(ABC):
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
log_online_quantization(vllm_config)
|
||||
log_model_inspection(model)
|
||||
|
||||
logger.debug("Loading weights on %s ...", load_device)
|
||||
@@ -92,6 +93,23 @@ def log_model_inspection(model: nn.Module) -> None:
|
||||
logger.info("vLLM model structure:\n%s", format_model_inspection(model))
|
||||
|
||||
|
||||
def log_online_quantization(vllm_config: VllmConfig) -> None:
|
||||
"""Log the online-quantized layer count and types, when applicable."""
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
|
||||
quant_config = vllm_config.quant_config
|
||||
if not isinstance(quant_config, OnlineQuantizationConfig):
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Quantized %d layers of types: %s",
|
||||
len(quant_config.quantized_layers),
|
||||
"; ".join(quant_config.quantized_layer_summaries),
|
||||
)
|
||||
|
||||
|
||||
def _has_online_quant(model: nn.Module):
|
||||
for module in model.modules():
|
||||
quant_method = getattr(module, "quant_method", None)
|
||||
|
||||
Reference in New Issue
Block a user