[Online quantization] Add targeted online quantization configuration based on user patterns (#51285)

This commit is contained in:
fxmarty-amd
2026-09-02 08:55:15 -04:00
committed by GitHub
parent 2a4e3cc3db
commit 7894394b07
11 changed files with 892 additions and 172 deletions
@@ -1,11 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
from types import MappingProxyType
from typing import Any
import torch
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
from vllm.config.quantization import (
_ONLINE_SHORTHANDS,
QuantizationConfigArgs,
QuantSpec,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
RoutedExperts,
@@ -47,6 +53,10 @@ from vllm.model_executor.layers.quantization.online.mxfp8 import (
from vllm.model_executor.layers.quantization.online.nvfp4 import (
Nvfp4OnlineMoEMethod,
)
from vllm.model_executor.layers.quantization.utils.config_utils import (
find_matching_patterns,
get_layer_name_after_index,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Static128BlockSym,
@@ -83,6 +93,39 @@ _ONLINE_MOE_METHODS: dict[QuantKey, type] = {
}
def _find_matching_targets(
prefix: str,
targets: Mapping[str, str],
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
) -> list[str]:
per_shard_matches = find_matching_patterns(
prefix, targets, fused_mapping, use_fnmatch=True
)
if all(len(matches) == 0 for matches in per_shard_matches):
return []
if any(len(matches) == 0 for matches in per_shard_matches):
raise ValueError(
f"Found unmatched shards for {prefix}: {per_shard_matches}. vLLM "
"requires all shards of a fused layer to match a target."
)
if any(len(matches) > 1 for matches in per_shard_matches):
raise ValueError(
f"Found multiple quantization_config.targets matches for the "
f"shards of {prefix}: {per_shard_matches}. Each shard may match "
"at most one target."
)
matched_patterns = [next(iter(matches)) for matches in per_shard_matches]
quant_key_strs = {targets[pattern] for pattern in matched_patterns}
if len(quant_key_strs) > 1:
raise ValueError(
f"Found different quantization_config.targets values for the "
f"shards of {prefix}: {matched_patterns}. vLLM requires all "
"shards of a fused layer to use the same target."
)
return [matched_patterns[0]]
class OnlineQuantizationConfig(QuantizationConfig):
"""Model-level config for online quantization (quantize fp16/bf16 weights
during model loading, without requiring a pre-quantized checkpoint)."""
@@ -92,14 +135,44 @@ class OnlineQuantizationConfig(QuantizationConfig):
args: QuantizationConfigArgs,
) -> None:
super().__init__()
if args.linear is None and args.moe is None:
if args.linear is None and args.moe is None and args.targets is None:
raise ValueError(
"OnlineQuantizationConfig requires at least one of "
"quantization_config.linear or quantization_config.moe "
"to be set."
"quantization_config.linear, quantization_config.moe, or "
"quantization_config.targets to be set."
)
self.args = args
self.ignored_layers: list[str] = args.ignore
self.quantized_layers: dict[str, tuple[str, str, str | None]] = {}
@property
def quantized_layer_summaries(self) -> list[str]:
counts: dict[tuple[str, str, str | None, str], int] = {}
for layer_name, (
source,
quant_key_str,
target_pattern,
) in self.quantized_layers.items():
key = (
get_layer_name_after_index(layer_name),
source,
target_pattern,
quant_key_str,
)
counts[key] = counts.get(key, 0) + 1
summaries = []
# Build summary entries as
# `self_attn.o_proj: 24 (from targets: re:.*self_attn\.o_proj, mxfp4`
for (layer_type, source, target_pattern, quant_key_str), count in sorted(
counts.items()
):
pattern_prefix = f"{target_pattern}, " if target_pattern else ""
summaries.append(
f"{layer_type}: {count} "
f"(from {source}: {pattern_prefix}{quant_key_str})"
)
return summaries
@classmethod
def get_name(cls) -> QuantizationMethods:
@@ -154,29 +227,101 @@ class OnlineQuantizationConfig(QuantizationConfig):
return cls(layer=layer)
return cls()
def _dispatch_target(
self, prefix: str, layer: torch.nn.Module
) -> "QuantizeMethodBase | None":
assert self.args.targets is not None
ignored = should_ignore_layer(
prefix,
ignore=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
use_fnmatch=True,
)
matches = _find_matching_targets(
prefix, self.args.targets, fused_mapping=self.packed_modules_mapping
)
if ignored and matches:
raise ValueError(
f"Layer {prefix} matches both quantization_config.ignore "
f"and quantization_config.targets ({matches}); a layer may "
f"not be referenced by both."
)
if ignored or not matches:
return None
if len(matches) > 1:
raise ValueError(
f"Layer {prefix} matches multiple quantization_config."
f"targets patterns: {matches}. Each layer may match at most "
f"one target."
)
quant_key_str = self.args.targets[matches[0]]
shorthand = _ONLINE_SHORTHANDS[quant_key_str]
if isinstance(layer, LinearBase):
quant_spec = shorthand.linear
table = _ONLINE_LINEAR_METHODS
elif isinstance(layer, RoutedExperts):
quant_spec = shorthand.moe
table = _ONLINE_MOE_METHODS
else:
raise ValueError(
f"Layer {prefix} was matched by quantization_config.targets "
f"({matches[0]}), but online quantization is not supported for "
f"{type(layer).__name__}."
)
if quant_spec is None:
raise ValueError(
f"targets pattern {matches[0]} = {quant_key_str} does "
f"not define a QuantSpec for {type(layer).__name__} layers "
f"(matched at {prefix})."
)
method = self._dispatch(quant_spec, table, layer)
if method is not None:
self.quantized_layers[prefix] = ("targets", quant_key_str, matches[0])
return method
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
# `targets` takes precedence over `moe` and `linear` and is exclusive.
if self.args.targets is not None:
method = self._dispatch_target(prefix, layer)
if method is not None:
return method
if isinstance(layer, LinearBase):
return UnquantizedLinearMethod()
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
return None
if isinstance(layer, LinearBase):
if should_ignore_layer(
prefix,
ignore=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
use_fnmatch=True,
):
return UnquantizedLinearMethod()
method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer)
return method if method is not None else UnquantizedLinearMethod()
if method is not None:
self.quantized_layers[prefix] = (
"linear",
str(self.args.linear),
None,
)
return method
return UnquantizedLinearMethod()
elif isinstance(layer, RoutedExperts):
if should_ignore_layer(
prefix,
ignore=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
use_fnmatch=True,
):
return UnquantizedFusedMoEMethod(layer.moe_config)
method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer)
return (
method
if method is not None
else UnquantizedFusedMoEMethod(layer.moe_config)
)
if method is not None:
self.quantized_layers[prefix] = ("moe", str(self.args.moe), None)
return method
return UnquantizedFusedMoEMethod(layer.moe_config)
return None