[Quantization] Rework quantization_config to use QuantKey and allow for activation override (#41566)

This commit is contained in:
Michael Goin
2026-05-13 16:58:32 -04:00
committed by GitHub
parent cca32d55a2
commit 8efd508204
16 changed files with 621 additions and 322 deletions
+138 -89
View File
@@ -1,126 +1,175 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
from typing import Any
from typing import Annotated, Any
from pydantic import Field, field_validator
from pydantic import Field, GetPydanticSchema, ValidationInfo, field_validator
from pydantic_core import core_schema
from vllm.config.utils import config
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
kInt8StaticChannelSym,
kMxfp4Dynamic,
kMxfp8Dynamic,
)
# User-facing names addressable from quantization_config.
QUANT_KEY_NAMES: dict[str, QuantKey] = {
"fp8_per_tensor_static": kFp8StaticTensorSym,
"fp8_per_tensor_dynamic": kFp8DynamicTensorSym,
"fp8_per_token": kFp8DynamicTokenSym,
"fp8_per_block_static": kFp8Static128BlockSym,
"fp8_per_block_dynamic": kFp8Dynamic128Sym,
"mxfp8": kMxfp8Dynamic,
"mxfp4": kMxfp4Dynamic,
"int8_per_channel_static": kInt8StaticChannelSym,
}
class OnlineQuantScheme(Enum):
"""Supported online quantization schemes."""
def _coerce_quant_key(v: Any) -> QuantKey | None:
if v is None or isinstance(v, QuantKey):
return v
if not isinstance(v, str):
raise TypeError(f"expected str or QuantKey, got {type(v).__name__}")
try:
return QUANT_KEY_NAMES[v]
except KeyError:
raise ValueError(
f"unknown quantization name {v!r}; "
f"expected one of {sorted(QUANT_KEY_NAMES)}"
) from None
# fp8, weights and activations scaled per-tensor
FP8_PER_TENSOR = "fp8_per_tensor"
# fp8, activations scaled in blocks of 1x128 elements, weights scaled in
# blocks of 128x128 elements (popularized by DeepSeek)
FP8_PER_BLOCK = "fp8_per_block"
# int8, weight-only per-channel quantization for MoE expert weights.
# Linear layers remain unquantized.
INT8_PER_CHANNEL_WEIGHT_ONLY = "int8_per_channel_weight_only"
# mxfp8, weights scaled in blocks of 1x32 elements (microscaling FP8)
MXFP8 = "mxfp8"
# Stop pydantic from introspecting QuantKey: it transitively contains a
# NamedTuple with `ClassVar[GroupShape]` declarations that pydantic refuses.
QuantKeyField = Annotated[
QuantKey | None,
GetPydanticSchema(
lambda _src, _handler: core_schema.no_info_plain_validator_function(
_coerce_quant_key
)
),
]
@config
class OnlineQuantizationConfigArgs:
"""Configuration for online quantization.
class QuantSpec:
"""Quantization spec for one layer kind (linear or MoE).
Controls how ``OnlineQuantizationConfig`` is applied to a model.
At least one of ``global_scheme``, ``linear_scheme_override``, or
``moe_scheme_override`` must be set.
`None` on either side means the method class falls back to its own default
(typically inherited from the checkpoint, or unquantized for online).
"""
global_scheme: OnlineQuantScheme | None = None
"""Quantization scheme applied to every supported layer."""
weight: QuantKeyField = None
"""Weight quantization key, or a name from QUANT_KEY_NAMES."""
linear_scheme_override: OnlineQuantScheme | None = None
"""Quantization scheme override for ``LinearBase`` layers."""
activation: QuantKeyField = None
"""Activation quantization key, or a name from QUANT_KEY_NAMES."""
moe_scheme_override: OnlineQuantScheme | None = None
"""Quantization scheme override for ``FusedMoE`` layers."""
@config
class QuantizationConfigArgs:
"""User-facing quantization configuration.
See `docs/features/quantization/online.md` for the schema and shorthand
string forms accepted on `linear` and `moe`.
"""
linear: QuantSpec | None = None
"""Spec applied to ``LinearBase`` layers."""
moe: QuantSpec | None = None
"""Spec applied to ``FusedMoE`` layers."""
ignore: list[str] = Field(default_factory=list)
"""Layers to skip quantization for. Supports exact names and regex
patterns with ``re:`` prefix (e.g. ``re:.*attn.*``), consistent with
compressed_tensors layer skipping."""
"""Layers to skip quantization for."""
@field_validator(
"global_scheme", "linear_scheme_override", "moe_scheme_override", mode="before"
)
@field_validator("linear", "moe", mode="before")
@classmethod
def _coerce_scheme(
cls, v: str | OnlineQuantScheme | None
) -> OnlineQuantScheme | None:
if isinstance(v, str):
return OnlineQuantScheme(v)
return v
def _coerce_spec(cls, v: Any, info: ValidationInfo) -> Any:
if not isinstance(v, str):
return v
field_name = info.field_name
assert field_name is not None
if v in _ONLINE_SHORTHANDS:
spec = getattr(_ONLINE_SHORTHANDS[v], field_name)
if spec is None:
raise ValueError(
f"online shorthand {v!r} does not define a {field_name} spec"
)
return spec
return QuantSpec(weight=_coerce_quant_key(v))
def resolve_online_quant_config(
# CLI shorthands accepted by `--quantization`. Each desugars to a full
# QuantizationConfigArgs; activation overrides go through quantization_config.
_ONLINE_SHORTHANDS: dict[str, QuantizationConfigArgs] = {
"fp8_per_tensor": QuantizationConfigArgs(
linear=QuantSpec(weight=kFp8StaticTensorSym),
moe=QuantSpec(weight=kFp8StaticTensorSym),
),
"fp8_per_block": QuantizationConfigArgs(
linear=QuantSpec(weight=kFp8Static128BlockSym),
moe=QuantSpec(weight=kFp8Static128BlockSym),
),
"mxfp8": QuantizationConfigArgs(
linear=QuantSpec(weight=kMxfp8Dynamic),
moe=QuantSpec(weight=kMxfp8Dynamic),
),
# INT8 weight-only on MoE; linear stays unquantized (no `linear` field).
"int8_per_channel_weight_only": QuantizationConfigArgs(
moe=QuantSpec(weight=kInt8StaticChannelSym),
),
}
# Names accepted by `--quantization`; "online" means "use quantization_config".
ONLINE_QUANT_SHORTHAND_NAMES: tuple[str, ...] = (
*_ONLINE_SHORTHANDS.keys(),
"online",
)
def resolve_quantization_config(
quantization: str | None,
quantization_config: dict[str, Any] | OnlineQuantizationConfigArgs | None,
) -> OnlineQuantizationConfigArgs | None:
"""Resolve online quant scheme shorthand into a quantization config.
quantization_config: dict[str, Any] | QuantizationConfigArgs | None,
) -> QuantizationConfigArgs | None:
"""Resolve `--quantization` shorthand and `--quantization-config` into a
QuantizationConfigArgs.
If ``quantization`` is an online quant scheme (e.g. ``'fp8_per_tensor'``),
ensures ``quantization_config`` has a matching ``global_scheme`` and casts
it to :class:`OnlineQuantizationConfigArgs` if needed.
`quantization` is a CLI shorthand that desugars into a base config via
`_ONLINE_SHORTHANDS`. `quantization_config` is a dict or pre-built args
object. When both are given, fields explicitly set in `quantization_config`
take precedence over the shorthand.
"""
online_quant_values = {s.value for s in OnlineQuantScheme}
valid_quantization_values = online_quant_values | {"online"}
if quantization not in valid_quantization_values:
if quantization is not None and quantization not in ONLINE_QUANT_SHORTHAND_NAMES:
if quantization_config is not None:
raise ValueError(
f"quantization_config is only supported when quantization "
f"is one of {sorted(valid_quantization_values)}, "
f"quantization_config is only supported when quantization is "
f"one of {sorted(ONLINE_QUANT_SHORTHAND_NAMES)}, "
f"got quantization={quantization!r}"
)
return None
if quantization in online_quant_values:
scheme = OnlineQuantScheme(quantization)
base = _ONLINE_SHORTHANDS.get(quantization) if quantization else None
if quantization_config is None:
quantization_config = {
"global_scheme": scheme.value,
}
elif isinstance(quantization_config, OnlineQuantizationConfigArgs):
if quantization_config.global_scheme is None:
quantization_config.global_scheme = scheme
elif quantization_config.global_scheme != scheme:
raise ValueError(
f"quantization={quantization!r} conflicts with "
f"quantization_config.global_scheme="
f"{quantization_config.global_scheme.value!r}. "
f"These must match when both are specified."
)
elif isinstance(quantization_config, dict):
existing = quantization_config.get("global_scheme")
if existing is None:
quantization_config["global_scheme"] = scheme.value
else:
# Coerce to enum for comparison
existing_scheme = (
OnlineQuantScheme(existing)
if isinstance(existing, str)
else existing
)
if existing_scheme != scheme:
raise ValueError(
f"quantization={quantization!r} conflicts "
f"with quantization_config"
f"['global_scheme']={existing!r}. "
f"These must match when both are specified."
)
if quantization_config is None:
return base
# Cast dict to OnlineQuantizationConfigArgs
if isinstance(quantization_config, dict):
quantization_config = OnlineQuantizationConfigArgs(**quantization_config)
quantization_config = QuantizationConfigArgs(**quantization_config)
return quantization_config
if base is None:
return quantization_config
return QuantizationConfigArgs(
linear=quantization_config.linear or base.linear,
moe=quantization_config.moe or base.moe,
ignore=quantization_config.ignore or base.ignore,
)