[Online quantization] Add targeted online quantization configuration based on user patterns (#51285)

This commit is contained in:
fxmarty-amd
2026-09-02 08:55:15 -04:00
committed by GitHub
parent 2a4e3cc3db
commit 7894394b07
11 changed files with 892 additions and 172 deletions
+74 -2
View File
@@ -3,7 +3,14 @@
from typing import Annotated, Any
from pydantic import Field, GetPydanticSchema, ValidationInfo, field_validator
import regex as re
from pydantic import (
Field,
GetPydanticSchema,
ValidationInfo,
field_validator,
model_validator,
)
from pydantic_core import core_schema
from vllm.config.utils import config
@@ -76,6 +83,21 @@ class QuantSpec:
activation: QuantKeyField = None
"""Activation quantization key, or a name from QUANT_KEY_NAMES."""
def __str__(self) -> str:
def quant_key_str(quant_key: QuantKey | None) -> str:
if quant_key is None:
return "None"
return next(
(
name
for name, known_quant_key in QUANT_KEY_NAMES.items()
if known_quant_key == quant_key
),
str(quant_key),
)
return quant_key_str(self.weight)
@config
class QuantizationConfigArgs:
@@ -92,7 +114,16 @@ class QuantizationConfigArgs:
"""Spec applied to ``FusedMoEFactory`` layers."""
ignore: list[str] = Field(default_factory=list)
"""Layers to skip quantization for."""
"""Layers to skip quantization for. Online quantization also supports
fnmatch-style patterns."""
targets: dict[str, str] | None = None
"""Per-layer online quantization overrides, keyed by exact layer name or
regex patterns with a `re:`, or fnmatch-style patterns for online
quantization, mapping to an online shorthand name (see
`_ONLINE_SHORTHANDS`). A layer that matches no pattern is left unquantized.
Mutually exclusive with `linear` and `moe`.
"""
@field_validator("linear", "moe", mode="before")
@classmethod
@@ -110,6 +141,46 @@ class QuantizationConfigArgs:
return spec
return QuantSpec(weight=_coerce_quant_key(v))
@field_validator("targets", mode="before")
@classmethod
def _validate_targets(cls, v: Any) -> Any:
if v is None:
return v
if not isinstance(v, dict):
raise TypeError(f"targets must be a dict, got {type(v).__name__}")
for pattern, shorthand in v.items():
if not isinstance(pattern, str):
raise ValueError(
f"targets keys must be strings, got {type(pattern).__name__}"
)
if not isinstance(shorthand, str) or shorthand not in _ONLINE_SHORTHANDS:
raise ValueError(
f"targets[{pattern}] = {shorthand} is not a valid "
f"online shorthand name; expected one of "
f"{sorted(_ONLINE_SHORTHANDS)}"
)
if pattern.startswith("re:"):
try:
re.compile(pattern[3:])
except re.error as e:
raise ValueError(
f"targets key {pattern} is not a valid regex: {e}"
) from e
return v
@model_validator(mode="after")
def _validate_targets_exclusivity(self) -> "QuantizationConfigArgs":
if self.targets is None:
return self
if self.linear is not None or self.moe is not None:
raise ValueError(
"quantization_config.targets is mutually exclusive with "
f"quantization_config.linear/moe, got "
f"targets={self.targets}, linear={self.linear}, "
f"moe={self.moe}."
)
return self
# CLI shorthands accepted by `--quantization`. Each desugars to a full
# QuantizationConfigArgs; activation overrides go through quantization_config.
@@ -191,4 +262,5 @@ def resolve_quantization_config(
linear=quantization_config.linear or base.linear,
moe=quantization_config.moe or base.moe,
ignore=quantization_config.ignore or base.ignore,
targets=quantization_config.targets or base.targets,
)