[Online quantization] Add targeted online quantization configuration based on user patterns (#51285)
This commit is contained in:
@@ -4,11 +4,14 @@
|
||||
from collections.abc import Iterable, Mapping
|
||||
from types import MappingProxyType
|
||||
|
||||
import regex as re
|
||||
from compressed_tensors import CompressionFormat
|
||||
from compressed_tensors.quantization import QuantizationStrategy
|
||||
from torch.nn import Module
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
find_matching_patterns,
|
||||
is_equal_or_regex_match,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
@@ -51,63 +54,20 @@ def should_ignore_layer(
|
||||
layer_name: str | None,
|
||||
ignore: Iterable[str] = tuple(),
|
||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
|
||||
use_fnmatch: bool = False,
|
||||
) -> bool:
|
||||
if layer_name is None:
|
||||
return False
|
||||
|
||||
# layer_name = model.layers.0.self_attn.qkv_proj
|
||||
# proj_name = qkv_proj
|
||||
proj_name = layer_name.split(".")[-1]
|
||||
|
||||
# Fused layers like gate_up_proj or qkv_proj will not be fused
|
||||
# in the safetensors checkpoint. So, we convert the name
|
||||
# from the fused version to unfused + check to make sure that
|
||||
# each shard of the fused layer has the same scheme.
|
||||
if proj_name in fused_mapping and layer_name not in ignore:
|
||||
shard_proj_names = fused_mapping[proj_name]
|
||||
|
||||
# Convert fused_name --> [shard_names]
|
||||
shard_names = [
|
||||
layer_name.replace(proj_name, shard_proj_name)
|
||||
for shard_proj_name in shard_proj_names
|
||||
]
|
||||
|
||||
# Layer should be ignored if shards are ignored.
|
||||
should_ignore_layer = None
|
||||
for shard_name in shard_names:
|
||||
should_ignore_shard = check_equal_or_regex_match(
|
||||
layer_name=shard_name, targets=ignore
|
||||
)
|
||||
|
||||
# If shard_idx=0, set layer ignore to match shard.
|
||||
if should_ignore_layer is None:
|
||||
should_ignore_layer = should_ignore_shard
|
||||
|
||||
# If shard_idx=1+ confirm scheme matches prior shards.
|
||||
elif should_ignore_shard != should_ignore_layer:
|
||||
raise ValueError(
|
||||
f"Found a different quantization schemes for "
|
||||
f"{shard_proj_names} in {layer_name}. vLLM "
|
||||
"requires all to use the same scheme."
|
||||
)
|
||||
|
||||
# Unfused layers like down_proj and o_proj will match
|
||||
# the safetensors checkpoint already.
|
||||
else:
|
||||
should_ignore_layer = check_equal_or_regex_match(
|
||||
layer_name=layer_name, targets=ignore
|
||||
per_shard_matches = find_matching_patterns(
|
||||
layer_name, ignore, fused_mapping, use_fnmatch=use_fnmatch
|
||||
)
|
||||
shards_ignored = [len(matches) > 0 for matches in per_shard_matches]
|
||||
if any(shards_ignored) and not all(shards_ignored):
|
||||
raise ValueError(
|
||||
f"Found different quantization schemes for the shards of "
|
||||
f"{layer_name}. vLLM requires all to use the same scheme."
|
||||
)
|
||||
|
||||
assert should_ignore_layer is not None
|
||||
return should_ignore_layer
|
||||
|
||||
|
||||
def check_equal_or_regex_match(layer_name: str, targets: Iterable[str]) -> bool:
|
||||
"""
|
||||
Checks whether a layer_name is exactly equal or a regex match for
|
||||
if target starts with 're:' to any target in list.
|
||||
"""
|
||||
return any(_is_equal_or_regex_match(layer_name, target) for target in targets)
|
||||
return all(shards_ignored)
|
||||
|
||||
|
||||
def find_matched_target(
|
||||
@@ -167,32 +127,11 @@ def _find_first_match(
|
||||
"""
|
||||
|
||||
for target in targets:
|
||||
if _is_equal_or_regex_match(value, target, check_contains=check_contains):
|
||||
if is_equal_or_regex_match(value, target, check_contains=check_contains):
|
||||
return target
|
||||
return None
|
||||
|
||||
|
||||
def _is_equal_or_regex_match(
|
||||
value: str, target: str, check_contains: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Checks whether a value is exactly equal or a regex match for target
|
||||
if target starts with 're:'. If check_contains is set to True,
|
||||
additionally checks if the target string is contained within the value.
|
||||
"""
|
||||
|
||||
if target.startswith("re:"):
|
||||
pattern = target[3:]
|
||||
if re.match(pattern, value):
|
||||
return True
|
||||
elif check_contains:
|
||||
if target.lower() in value.lower():
|
||||
return True
|
||||
elif target == value:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _match_fused_layer(
|
||||
layer_name: str,
|
||||
target_layers: Iterable[str],
|
||||
@@ -230,7 +169,7 @@ def _match_fused_layer(
|
||||
unfused_matches: list[str | None] = []
|
||||
for unfused in unfused_paths:
|
||||
for target in target_layers:
|
||||
if _is_equal_or_regex_match(unfused, target):
|
||||
if is_equal_or_regex_match(unfused, target):
|
||||
unfused_matches.append(target)
|
||||
break
|
||||
else:
|
||||
|
||||
@@ -1,11 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
|
||||
from vllm.config.quantization import (
|
||||
_ONLINE_SHORTHANDS,
|
||||
QuantizationConfigArgs,
|
||||
QuantSpec,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
@@ -47,6 +53,10 @@ from vllm.model_executor.layers.quantization.online.mxfp8 import (
|
||||
from vllm.model_executor.layers.quantization.online.nvfp4 import (
|
||||
Nvfp4OnlineMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
find_matching_patterns,
|
||||
get_layer_name_after_index,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Static128BlockSym,
|
||||
@@ -83,6 +93,39 @@ _ONLINE_MOE_METHODS: dict[QuantKey, type] = {
|
||||
}
|
||||
|
||||
|
||||
def _find_matching_targets(
|
||||
prefix: str,
|
||||
targets: Mapping[str, str],
|
||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
|
||||
) -> list[str]:
|
||||
per_shard_matches = find_matching_patterns(
|
||||
prefix, targets, fused_mapping, use_fnmatch=True
|
||||
)
|
||||
if all(len(matches) == 0 for matches in per_shard_matches):
|
||||
return []
|
||||
if any(len(matches) == 0 for matches in per_shard_matches):
|
||||
raise ValueError(
|
||||
f"Found unmatched shards for {prefix}: {per_shard_matches}. vLLM "
|
||||
"requires all shards of a fused layer to match a target."
|
||||
)
|
||||
if any(len(matches) > 1 for matches in per_shard_matches):
|
||||
raise ValueError(
|
||||
f"Found multiple quantization_config.targets matches for the "
|
||||
f"shards of {prefix}: {per_shard_matches}. Each shard may match "
|
||||
"at most one target."
|
||||
)
|
||||
|
||||
matched_patterns = [next(iter(matches)) for matches in per_shard_matches]
|
||||
quant_key_strs = {targets[pattern] for pattern in matched_patterns}
|
||||
if len(quant_key_strs) > 1:
|
||||
raise ValueError(
|
||||
f"Found different quantization_config.targets values for the "
|
||||
f"shards of {prefix}: {matched_patterns}. vLLM requires all "
|
||||
"shards of a fused layer to use the same target."
|
||||
)
|
||||
return [matched_patterns[0]]
|
||||
|
||||
|
||||
class OnlineQuantizationConfig(QuantizationConfig):
|
||||
"""Model-level config for online quantization (quantize fp16/bf16 weights
|
||||
during model loading, without requiring a pre-quantized checkpoint)."""
|
||||
@@ -92,14 +135,44 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
args: QuantizationConfigArgs,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if args.linear is None and args.moe is None:
|
||||
if args.linear is None and args.moe is None and args.targets is None:
|
||||
raise ValueError(
|
||||
"OnlineQuantizationConfig requires at least one of "
|
||||
"quantization_config.linear or quantization_config.moe "
|
||||
"to be set."
|
||||
"quantization_config.linear, quantization_config.moe, or "
|
||||
"quantization_config.targets to be set."
|
||||
)
|
||||
self.args = args
|
||||
self.ignored_layers: list[str] = args.ignore
|
||||
self.quantized_layers: dict[str, tuple[str, str, str | None]] = {}
|
||||
|
||||
@property
|
||||
def quantized_layer_summaries(self) -> list[str]:
|
||||
counts: dict[tuple[str, str, str | None, str], int] = {}
|
||||
for layer_name, (
|
||||
source,
|
||||
quant_key_str,
|
||||
target_pattern,
|
||||
) in self.quantized_layers.items():
|
||||
key = (
|
||||
get_layer_name_after_index(layer_name),
|
||||
source,
|
||||
target_pattern,
|
||||
quant_key_str,
|
||||
)
|
||||
counts[key] = counts.get(key, 0) + 1
|
||||
|
||||
summaries = []
|
||||
# Build summary entries as
|
||||
# `self_attn.o_proj: 24 (from targets: re:.*self_attn\.o_proj, mxfp4`
|
||||
for (layer_type, source, target_pattern, quant_key_str), count in sorted(
|
||||
counts.items()
|
||||
):
|
||||
pattern_prefix = f"{target_pattern}, " if target_pattern else ""
|
||||
summaries.append(
|
||||
f"{layer_type}: {count} "
|
||||
f"(from {source}: {pattern_prefix}{quant_key_str})"
|
||||
)
|
||||
return summaries
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
@@ -154,29 +227,101 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
return cls(layer=layer)
|
||||
return cls()
|
||||
|
||||
def _dispatch_target(
|
||||
self, prefix: str, layer: torch.nn.Module
|
||||
) -> "QuantizeMethodBase | None":
|
||||
assert self.args.targets is not None
|
||||
ignored = should_ignore_layer(
|
||||
prefix,
|
||||
ignore=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
use_fnmatch=True,
|
||||
)
|
||||
matches = _find_matching_targets(
|
||||
prefix, self.args.targets, fused_mapping=self.packed_modules_mapping
|
||||
)
|
||||
if ignored and matches:
|
||||
raise ValueError(
|
||||
f"Layer {prefix} matches both quantization_config.ignore "
|
||||
f"and quantization_config.targets ({matches}); a layer may "
|
||||
f"not be referenced by both."
|
||||
)
|
||||
if ignored or not matches:
|
||||
return None
|
||||
if len(matches) > 1:
|
||||
raise ValueError(
|
||||
f"Layer {prefix} matches multiple quantization_config."
|
||||
f"targets patterns: {matches}. Each layer may match at most "
|
||||
f"one target."
|
||||
)
|
||||
quant_key_str = self.args.targets[matches[0]]
|
||||
shorthand = _ONLINE_SHORTHANDS[quant_key_str]
|
||||
if isinstance(layer, LinearBase):
|
||||
quant_spec = shorthand.linear
|
||||
table = _ONLINE_LINEAR_METHODS
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
quant_spec = shorthand.moe
|
||||
table = _ONLINE_MOE_METHODS
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Layer {prefix} was matched by quantization_config.targets "
|
||||
f"({matches[0]}), but online quantization is not supported for "
|
||||
f"{type(layer).__name__}."
|
||||
)
|
||||
if quant_spec is None:
|
||||
raise ValueError(
|
||||
f"targets pattern {matches[0]} = {quant_key_str} does "
|
||||
f"not define a QuantSpec for {type(layer).__name__} layers "
|
||||
f"(matched at {prefix})."
|
||||
)
|
||||
method = self._dispatch(quant_spec, table, layer)
|
||||
if method is not None:
|
||||
self.quantized_layers[prefix] = ("targets", quant_key_str, matches[0])
|
||||
return method
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
# `targets` takes precedence over `moe` and `linear` and is exclusive.
|
||||
if self.args.targets is not None:
|
||||
method = self._dispatch_target(prefix, layer)
|
||||
if method is not None:
|
||||
return method
|
||||
|
||||
if isinstance(layer, LinearBase):
|
||||
return UnquantizedLinearMethod()
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
return None
|
||||
|
||||
if isinstance(layer, LinearBase):
|
||||
if should_ignore_layer(
|
||||
prefix,
|
||||
ignore=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
use_fnmatch=True,
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer)
|
||||
return method if method is not None else UnquantizedLinearMethod()
|
||||
if method is not None:
|
||||
self.quantized_layers[prefix] = (
|
||||
"linear",
|
||||
str(self.args.linear),
|
||||
None,
|
||||
)
|
||||
return method
|
||||
return UnquantizedLinearMethod()
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
if should_ignore_layer(
|
||||
prefix,
|
||||
ignore=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
use_fnmatch=True,
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer)
|
||||
return (
|
||||
method
|
||||
if method is not None
|
||||
else UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
)
|
||||
if method is not None:
|
||||
self.quantized_layers[prefix] = ("moe", str(self.args.moe), None)
|
||||
return method
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
return None
|
||||
|
||||
@@ -5,9 +5,12 @@ from collections.abc import Iterable, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
find_matching_patterns,
|
||||
)
|
||||
|
||||
QuarkQTensorHint = dict[str, Any] | list[dict[str, Any]] | None
|
||||
|
||||
|
||||
@@ -49,80 +52,18 @@ def should_ignore_layer(
|
||||
):
|
||||
return True
|
||||
|
||||
# layer_name = model.layers.0.self_attn.qkv_proj
|
||||
# proj_name = qkv_proj
|
||||
proj_name = layer_name.split(".")[-1]
|
||||
|
||||
# Fused layers like gate_up_proj or qkv_proj will not be fused
|
||||
# in the safetensors checkpoint. So, we convert the name
|
||||
# from the fused version to unfused + check to make sure that
|
||||
# each shard of the fused layer has the same scheme.
|
||||
if proj_name in fused_mapping:
|
||||
shard_proj_names = fused_mapping[proj_name]
|
||||
|
||||
# Convert fused_name --> [shard_names]
|
||||
shard_names = [
|
||||
layer_name.replace(proj_name, shard_proj_name)
|
||||
for shard_proj_name in shard_proj_names
|
||||
]
|
||||
|
||||
# Layer should be ignored if shards are ignored.
|
||||
should_ignore_layer = None
|
||||
for shard_name in shard_names:
|
||||
should_ignore_shard = check_equal_or_regex_match(
|
||||
layer_name=shard_name, targets=ignore
|
||||
)
|
||||
|
||||
# If shard_idx=0, set layer ignore to match shard.
|
||||
if should_ignore_layer is None:
|
||||
should_ignore_layer = should_ignore_shard
|
||||
|
||||
# If shard_idx=1+ confirm scheme matches prior shards.
|
||||
elif should_ignore_shard != should_ignore_layer:
|
||||
raise ValueError(
|
||||
f"Found a different quantization schemes for "
|
||||
f"{shard_proj_names} in {layer_name}. vLLM "
|
||||
"requires all to use the same scheme."
|
||||
)
|
||||
|
||||
# Unfused layers like down_proj and o_proj will match
|
||||
# the safetensors checkpoint already.
|
||||
else:
|
||||
should_ignore_layer = check_equal_or_regex_match(
|
||||
layer_name=layer_name, targets=ignore
|
||||
# A direct fused-layer pattern takes precedence over expansion. For
|
||||
# model.layers.0.self_attn.qkv_proj,
|
||||
# ignore=["re:.*qkv_proj.*"] yields [{"re:.*qkv_proj.*"}]. In contrast,
|
||||
# ignore=["re:.*[qkv]_proj"] yields one matching set per expanded shard.
|
||||
per_shard_matches = find_matching_patterns(layer_name, ignore, fused_mapping)
|
||||
shards_ignored = [len(matches) > 0 for matches in per_shard_matches]
|
||||
if any(shards_ignored) and not all(shards_ignored):
|
||||
raise ValueError(
|
||||
f"Found different quantization schemes for the shards of "
|
||||
f"{layer_name}. vLLM requires all to use the same scheme."
|
||||
)
|
||||
|
||||
assert should_ignore_layer is not None
|
||||
return should_ignore_layer
|
||||
|
||||
|
||||
def check_equal_or_regex_match(layer_name: str, targets: Iterable[str]) -> bool:
|
||||
"""
|
||||
Checks whether a layer_name is exactly equal or a regex match for
|
||||
if target starts with 're:' to any target in list.
|
||||
"""
|
||||
return any(_is_equal_or_regex_match(layer_name, target) for target in targets)
|
||||
|
||||
|
||||
def _is_equal_or_regex_match(
|
||||
value: str, target: str, check_contains: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Checks whether a value is exactly equal or a regex match for target
|
||||
if target starts with 're:'. If check_contains is set to True,
|
||||
additionally checks if the target string is contained within the value.
|
||||
"""
|
||||
|
||||
if target.startswith("re:"):
|
||||
pattern = target[3:]
|
||||
if re.match(pattern, value):
|
||||
return True
|
||||
elif check_contains:
|
||||
if target.lower() in value.lower():
|
||||
return True
|
||||
elif target == value:
|
||||
return True
|
||||
return False
|
||||
return all(shards_ignored)
|
||||
|
||||
|
||||
# utility for tensor dims > 2 cases
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from fnmatch import fnmatch
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
@@ -244,3 +247,76 @@ def is_shared_expert_quant_fse_compatible(
|
||||
"shared-expert FSE quantization compatibility is not implemented for "
|
||||
f"{type(quant_config).__name__}",
|
||||
)
|
||||
|
||||
|
||||
def find_matching_patterns(
|
||||
layer_name: str,
|
||||
patterns: Iterable[str],
|
||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
|
||||
use_fnmatch: bool = False,
|
||||
) -> list[set[str]]:
|
||||
"""Return matching patterns for a layer or each shard of a fused layer.
|
||||
|
||||
A pattern matching the fused layer directly takes precedence. Otherwise,
|
||||
return one set of matching patterns for every shard.
|
||||
"""
|
||||
patterns = list(patterns)
|
||||
matches = [
|
||||
pattern
|
||||
for pattern in patterns
|
||||
if is_equal_or_regex_match(layer_name, pattern, use_fnmatch=use_fnmatch)
|
||||
]
|
||||
if matches:
|
||||
return [set(matches)]
|
||||
|
||||
proj_name = layer_name.split(".")[-1]
|
||||
if proj_name not in fused_mapping:
|
||||
return [set()]
|
||||
|
||||
shard_names = [
|
||||
layer_name.replace(proj_name, shard_proj_name)
|
||||
for shard_proj_name in fused_mapping[proj_name]
|
||||
]
|
||||
per_shard_matches = [
|
||||
{
|
||||
pattern
|
||||
for pattern in patterns
|
||||
if is_equal_or_regex_match(shard_name, pattern, use_fnmatch=use_fnmatch)
|
||||
}
|
||||
for shard_name in shard_names
|
||||
]
|
||||
return per_shard_matches
|
||||
|
||||
|
||||
def get_layer_name_after_index(layer_name: str) -> str:
|
||||
"""Return the suffix following the final numeric component of a layer name."""
|
||||
parts = layer_name.split(".")
|
||||
for index in range(len(parts) - 1, -1, -1):
|
||||
if parts[index].isdigit():
|
||||
return ".".join(parts[index + 1 :])
|
||||
return layer_name
|
||||
|
||||
|
||||
def is_equal_or_regex_match(
|
||||
value: str,
|
||||
target: str,
|
||||
check_contains: bool = False,
|
||||
use_fnmatch: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
Checks whether a value is exactly equal or a regex match for target
|
||||
if target starts with 're:'. If check_contains is set to True,
|
||||
additionally checks if the target string is contained within the value.
|
||||
If use_fnmatch is set, supports shell-style patterns in target.
|
||||
"""
|
||||
|
||||
if target.startswith("re:"):
|
||||
pattern = target[3:]
|
||||
if re.match(pattern, value):
|
||||
return True
|
||||
elif check_contains:
|
||||
if target.lower() in value.lower():
|
||||
return True
|
||||
elif target == value or use_fnmatch and fnmatch(value, target):
|
||||
return True
|
||||
return False
|
||||
|
||||
Reference in New Issue
Block a user