[Online quantization] Add targeted online quantization configuration based on user patterns (#51285)
This commit is contained in:
@@ -3,7 +3,14 @@
|
||||
|
||||
from typing import Annotated, Any
|
||||
|
||||
from pydantic import Field, GetPydanticSchema, ValidationInfo, field_validator
|
||||
import regex as re
|
||||
from pydantic import (
|
||||
Field,
|
||||
GetPydanticSchema,
|
||||
ValidationInfo,
|
||||
field_validator,
|
||||
model_validator,
|
||||
)
|
||||
from pydantic_core import core_schema
|
||||
|
||||
from vllm.config.utils import config
|
||||
@@ -76,6 +83,21 @@ class QuantSpec:
|
||||
activation: QuantKeyField = None
|
||||
"""Activation quantization key, or a name from QUANT_KEY_NAMES."""
|
||||
|
||||
def __str__(self) -> str:
|
||||
def quant_key_str(quant_key: QuantKey | None) -> str:
|
||||
if quant_key is None:
|
||||
return "None"
|
||||
return next(
|
||||
(
|
||||
name
|
||||
for name, known_quant_key in QUANT_KEY_NAMES.items()
|
||||
if known_quant_key == quant_key
|
||||
),
|
||||
str(quant_key),
|
||||
)
|
||||
|
||||
return quant_key_str(self.weight)
|
||||
|
||||
|
||||
@config
|
||||
class QuantizationConfigArgs:
|
||||
@@ -92,7 +114,16 @@ class QuantizationConfigArgs:
|
||||
"""Spec applied to ``FusedMoEFactory`` layers."""
|
||||
|
||||
ignore: list[str] = Field(default_factory=list)
|
||||
"""Layers to skip quantization for."""
|
||||
"""Layers to skip quantization for. Online quantization also supports
|
||||
fnmatch-style patterns."""
|
||||
|
||||
targets: dict[str, str] | None = None
|
||||
"""Per-layer online quantization overrides, keyed by exact layer name or
|
||||
regex patterns with a `re:`, or fnmatch-style patterns for online
|
||||
quantization, mapping to an online shorthand name (see
|
||||
`_ONLINE_SHORTHANDS`). A layer that matches no pattern is left unquantized.
|
||||
Mutually exclusive with `linear` and `moe`.
|
||||
"""
|
||||
|
||||
@field_validator("linear", "moe", mode="before")
|
||||
@classmethod
|
||||
@@ -110,6 +141,46 @@ class QuantizationConfigArgs:
|
||||
return spec
|
||||
return QuantSpec(weight=_coerce_quant_key(v))
|
||||
|
||||
@field_validator("targets", mode="before")
|
||||
@classmethod
|
||||
def _validate_targets(cls, v: Any) -> Any:
|
||||
if v is None:
|
||||
return v
|
||||
if not isinstance(v, dict):
|
||||
raise TypeError(f"targets must be a dict, got {type(v).__name__}")
|
||||
for pattern, shorthand in v.items():
|
||||
if not isinstance(pattern, str):
|
||||
raise ValueError(
|
||||
f"targets keys must be strings, got {type(pattern).__name__}"
|
||||
)
|
||||
if not isinstance(shorthand, str) or shorthand not in _ONLINE_SHORTHANDS:
|
||||
raise ValueError(
|
||||
f"targets[{pattern}] = {shorthand} is not a valid "
|
||||
f"online shorthand name; expected one of "
|
||||
f"{sorted(_ONLINE_SHORTHANDS)}"
|
||||
)
|
||||
if pattern.startswith("re:"):
|
||||
try:
|
||||
re.compile(pattern[3:])
|
||||
except re.error as e:
|
||||
raise ValueError(
|
||||
f"targets key {pattern} is not a valid regex: {e}"
|
||||
) from e
|
||||
return v
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_targets_exclusivity(self) -> "QuantizationConfigArgs":
|
||||
if self.targets is None:
|
||||
return self
|
||||
if self.linear is not None or self.moe is not None:
|
||||
raise ValueError(
|
||||
"quantization_config.targets is mutually exclusive with "
|
||||
f"quantization_config.linear/moe, got "
|
||||
f"targets={self.targets}, linear={self.linear}, "
|
||||
f"moe={self.moe}."
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
# CLI shorthands accepted by `--quantization`. Each desugars to a full
|
||||
# QuantizationConfigArgs; activation overrides go through quantization_config.
|
||||
@@ -191,4 +262,5 @@ def resolve_quantization_config(
|
||||
linear=quantization_config.linear or base.linear,
|
||||
moe=quantization_config.moe or base.moe,
|
||||
ignore=quantization_config.ignore or base.ignore,
|
||||
targets=quantization_config.targets or base.targets,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user