[Quantization] Rework quantization_config to use QuantKey and allow for activation override (#41566)

This commit is contained in:
Michael Goin
2026-05-13 16:58:32 -04:00
committed by GitHub
parent cca32d55a2
commit 8efd508204
16 changed files with 621 additions and 322 deletions
+45 -3
View File
@@ -42,18 +42,59 @@ vllm serve meta-llama/Llama-3.1-8B --quantization mxfp8
For fine-grained control, use a `quantization_config` dictionary.
### Schema
```yaml
quantization_config:
linear:
weight: <name> # see QUANT_KEY_NAMES in vllm/config/quantization.py
activation: <name>
moe:
weight: <name>
activation: <name>
ignore: [<layer-name-or-regex>, ...]
```
`linear` and `moe` accept a full `{weight, activation}` dict, or a bare
string. A string resolves first against the `--quantization` shorthands
(taking the matching layer-kind slot), then against `QUANT_KEY_NAMES` as a
weight name. Unset fields fall back to the `--quantization` shorthand's
defaults, or for already-quantized checkpoints to whatever the checkpoint
declares.
The CLI accepts the same shape as JSON or as dotted keys:
```bash
vllm serve <model> --quantization-config '{"moe":{"activation":"mxfp8"}}'
vllm serve <model> --quantization-config.moe.activation mxfp8
```
### Activation overrides on already-quantized checkpoints
For checkpoint-quantized models, `quantization_config` lets you pick an
activation format independently of the baked-in weights. The supported
overrides are checkpoint-specific; today this is wired up for MXFP4 MoE
checkpoints (gpt-oss) where you can opt into FP8 activations:
```bash
vllm serve openai/gpt-oss-20b --quantization-config.moe.activation mxfp8
```
Combine with `--moe-backend` to pin a specific kernel family.
### Separate Schemes for Dense and MoE Layers
You can apply different quantization schemes to dense linear layers and MoE expert layers:
You can apply different quantization schemes to dense linear layers and MoE expert layers via the `linear` and `moe` fields. Each accepts either a full spec dict, or a bare string naming an online shorthand (e.g. `"fp8_per_block"`) or weight format (e.g. `"fp8_per_block_static"`); fields not set fall back to the shorthand defaults.
```python
from vllm import LLM
# Linear: per-block FP8; MoE: per-tensor FP8 (inherited from the shorthand)
llm = LLM(
"ibm-granite/granite-3.0-1b-a400m-base",
quantization="fp8_per_tensor",
quantization_config={
"linear_scheme_override": "fp8_per_block",
"linear": "fp8_per_block",
},
)
```
@@ -63,11 +104,12 @@ Or,
```python
from vllm import LLM
# Linear: per-tensor FP8 (inherited); MoE: per-block FP8
llm = LLM(
"ibm-granite/granite-3.0-1b-a400m-base",
quantization="fp8_per_tensor",
quantization_config={
"moe_scheme_override": "fp8_per_block",
"moe": "fp8_per_block",
},
)
```
-7
View File
@@ -103,13 +103,6 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
if not supports_trtllm_attention():
matches = matches._replace(attn_quant_fusion=0)
# TODO: remove this after finishing migration from envs to model kwargs
if model_name == "openai/gpt-oss-20b":
from .common import is_blackwell
if is_blackwell():
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "1")
# Disable, compile cache to make sure custom passes run.
# Otherwise, we can't verify fusion happened through the logs.
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
+5
View File
@@ -214,4 +214,9 @@ gpt_oss_20b = ModelFusionInfo(
sequence_parallel=n_layers * 2 + 1,
async_tp=n_layers * 2,
),
model_kwargs=(
{"quantization_config": {"moe": {"activation": "mxfp8"}}}
if is_blackwell()
else {}
),
)
@@ -3,6 +3,4 @@
model_name: "openai/gpt-oss-20b"
metric_threshold: 0.568
reasoning_effort: "low"
server_args: "--tensor-parallel-size 2"
env:
VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS: "1"
server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_cutlass --quantization-config.moe.activation mxfp8"
@@ -3,6 +3,4 @@
model_name: "openai/gpt-oss-20b"
metric_threshold: 0.568
reasoning_effort: "low"
server_args: "--tensor-parallel-size 2"
env:
VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: "1"
server_args: "--tensor-parallel-size 2 --quantization-config.moe.activation mxfp8"
+18 -4
View File
@@ -190,13 +190,27 @@ def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch):
def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", "1")
can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT)
can_initialize(
"openai/gpt-oss-20b",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=[
"--moe-backend",
"flashinfer_cutlass",
"--quantization-config.moe.activation",
"mxfp8",
],
)
def test_gptoss_mxfp4mxfp8_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "1")
can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT)
can_initialize(
"openai/gpt-oss-20b",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=[
"--quantization-config.moe.activation",
"mxfp8",
],
)
def test_gptoss_eager(monkeypatch: pytest.MonkeyPatch):
+3 -4
View File
@@ -40,13 +40,12 @@ from vllm.platforms import current_platform
Fp8PerBlockOnlineLinearMethod,
Fp8PerBlockOnlineMoEMethod,
),
# quantization='online with linear_scheme_override and
# moe_scheme_override
# quantization='online' with per-layer-kind overrides
(
"online",
{
"linear_scheme_override": "fp8_per_block",
"moe_scheme_override": "fp8_per_tensor",
"linear": "fp8_per_block",
"moe": "fp8_per_tensor",
},
Fp8PerBlockOnlineLinearMethod,
Fp8PerTensorOnlineMoEMethod,
@@ -0,0 +1,134 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for QuantizationConfigArgs parsing."""
import pytest
from vllm.config.quantization import (
QUANT_KEY_NAMES,
QuantizationConfigArgs,
QuantSpec,
resolve_quantization_config,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
kInt8StaticChannelSym,
kMxfp8Dynamic,
)
# ---- QuantSpec ------------------------------------------------------------
def test_quant_spec_resolves_string_to_quant_key():
spec = QuantSpec(weight="mxfp8", activation="fp8_per_token")
assert spec.weight == kMxfp8Dynamic
assert spec.activation == kFp8DynamicTokenSym
def test_quant_spec_accepts_quant_key_directly():
spec = QuantSpec(weight=kFp8StaticTensorSym)
assert spec.weight is kFp8StaticTensorSym
assert spec.activation is None
def test_quant_spec_rejects_unknown_name():
with pytest.raises(ValueError, match="unknown quantization name"):
QuantSpec(weight="not_a_real_format")
# ---- QuantizationConfigArgs string shorthand on linear/moe ----------------
def test_args_linear_string_resolves_via_quant_key_names():
# A bare QUANT_KEY_NAMES entry desugars to QuantSpec(weight=<key>).
args = QuantizationConfigArgs(linear="fp8_per_block_static")
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
assert args.moe is None
def test_args_moe_string_resolves_via_online_shorthand():
# An online-shorthand name pulls the matching slot from _ONLINE_SHORTHANDS
# (so `linear: "fp8_per_block"` and `moe: "fp8_per_block"` produce the
# same per-layer-kind spec the `--quantization fp8_per_block` shorthand
# would).
args = QuantizationConfigArgs(moe="fp8_per_block")
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
def test_args_string_shorthand_missing_slot_raises():
# int8_per_channel_weight_only sets only `moe`; using it on `linear`
# has no defined spec and should raise rather than silently no-op.
with pytest.raises(ValueError, match="does not define a linear spec"):
QuantizationConfigArgs(linear="int8_per_channel_weight_only")
def test_args_accepts_dict_form():
args = QuantizationConfigArgs(moe={"activation": "mxfp8"})
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
# ---- resolve_quantization_config -----------------------------------------
def test_resolve_shorthand_only_populates_both_slots():
args = resolve_quantization_config("fp8_per_block", None)
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
def test_resolve_int8_shorthand_leaves_linear_unset():
# int8_per_channel_weight_only is MoE-only; linear stays None so that
# OnlineQuantizationConfig leaves Linear layers in full precision.
args = resolve_quantization_config("int8_per_channel_weight_only", None)
assert args.linear is None
assert args.moe == QuantSpec(weight=kInt8StaticChannelSym)
def test_resolve_quantization_config_only():
# When only `quantization_config` is given (e.g. for an already-quantized
# checkpoint that needs an activation override), it's returned as-is.
args = resolve_quantization_config(None, {"moe": {"activation": "mxfp8"}})
assert args.linear is None
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
def test_resolve_merges_explicit_over_shorthand():
# Explicit linear in quantization_config wins; moe falls back to the
# shorthand's slot.
args = resolve_quantization_config(
"fp8_per_tensor",
{"linear": "fp8_per_block"},
)
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
assert args.moe == QuantSpec(weight=kFp8StaticTensorSym)
def test_resolve_rejects_quantization_config_with_non_shorthand_quant():
# If --quantization names something other than an online shorthand,
# quantization_config is not allowed via this path (checkpoint quant
# paths read it directly off ModelConfig instead).
with pytest.raises(ValueError, match="quantization_config is only supported"):
resolve_quantization_config("gptq", {"linear": "fp8_per_block"})
# ---- QUANT_KEY_NAMES coverage --------------------------------------------
def test_quant_key_names_round_trip():
# Every advertised name should round-trip through QuantSpec without error
# and produce the same QuantKey it maps to.
for name, expected in QUANT_KEY_NAMES.items():
assert QuantSpec(weight=name).weight == expected, name
assert QuantSpec(activation=name).activation == expected, name
def test_static_block_weight_paired_with_dynamic_block_activation():
# The block-FP8 shorthand pair: 128x128 static weights + 1x128 dynamic
# activations. Pinning this so renames in QUANT_KEY_NAMES don't quietly
# rewire the kernel dispatch.
spec = QuantSpec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
assert spec.weight == kFp8Static128BlockSym
assert spec.activation == kFp8Dynamic128Sym
+6 -5
View File
@@ -21,7 +21,7 @@ from vllm.config.multimodal import (
MultiModalConfig,
)
from vllm.config.pooler import PoolerConfig
from vllm.config.quantization import OnlineQuantizationConfigArgs
from vllm.config.quantization import QuantizationConfigArgs
from vllm.config.scheduler import RunnerType
from vllm.config.utils import config, getattr_iter
from vllm.logger import init_logger
@@ -206,10 +206,11 @@ class ModelConfig:
`quantization_config` attribute in the model config file. If that is
`None`, we assume the model weights are not quantized and use `dtype` to
determine the data type of the weights."""
quantization_config: dict[str, Any] | OnlineQuantizationConfigArgs | None = None
"""Arguments for online quantization.
Auto-created when `quantization` equals to one of the string values of
the `OnlineQuantScheme` enum."""
quantization_config: dict[str, Any] | QuantizationConfigArgs | None = None
"""User-facing quantization configuration. Carries per-layer-kind specs
(linear, moe) and ignore patterns; see :class:`QuantizationConfigArgs`.
Auto-populated from the matching online shorthand when `quantization` is
one of the values in `ONLINE_QUANT_SHORTHAND_NAMES`."""
allow_deprecated_quantization: bool = False
"""Whether to allow deprecated quantization methods."""
enforce_eager: bool = False
+138 -89
View File
@@ -1,126 +1,175 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
from typing import Any
from typing import Annotated, Any
from pydantic import Field, field_validator
from pydantic import Field, GetPydanticSchema, ValidationInfo, field_validator
from pydantic_core import core_schema
from vllm.config.utils import config
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
kInt8StaticChannelSym,
kMxfp4Dynamic,
kMxfp8Dynamic,
)
# User-facing names addressable from quantization_config.
QUANT_KEY_NAMES: dict[str, QuantKey] = {
"fp8_per_tensor_static": kFp8StaticTensorSym,
"fp8_per_tensor_dynamic": kFp8DynamicTensorSym,
"fp8_per_token": kFp8DynamicTokenSym,
"fp8_per_block_static": kFp8Static128BlockSym,
"fp8_per_block_dynamic": kFp8Dynamic128Sym,
"mxfp8": kMxfp8Dynamic,
"mxfp4": kMxfp4Dynamic,
"int8_per_channel_static": kInt8StaticChannelSym,
}
class OnlineQuantScheme(Enum):
"""Supported online quantization schemes."""
def _coerce_quant_key(v: Any) -> QuantKey | None:
if v is None or isinstance(v, QuantKey):
return v
if not isinstance(v, str):
raise TypeError(f"expected str or QuantKey, got {type(v).__name__}")
try:
return QUANT_KEY_NAMES[v]
except KeyError:
raise ValueError(
f"unknown quantization name {v!r}; "
f"expected one of {sorted(QUANT_KEY_NAMES)}"
) from None
# fp8, weights and activations scaled per-tensor
FP8_PER_TENSOR = "fp8_per_tensor"
# fp8, activations scaled in blocks of 1x128 elements, weights scaled in
# blocks of 128x128 elements (popularized by DeepSeek)
FP8_PER_BLOCK = "fp8_per_block"
# int8, weight-only per-channel quantization for MoE expert weights.
# Linear layers remain unquantized.
INT8_PER_CHANNEL_WEIGHT_ONLY = "int8_per_channel_weight_only"
# mxfp8, weights scaled in blocks of 1x32 elements (microscaling FP8)
MXFP8 = "mxfp8"
# Stop pydantic from introspecting QuantKey: it transitively contains a
# NamedTuple with `ClassVar[GroupShape]` declarations that pydantic refuses.
QuantKeyField = Annotated[
QuantKey | None,
GetPydanticSchema(
lambda _src, _handler: core_schema.no_info_plain_validator_function(
_coerce_quant_key
)
),
]
@config
class OnlineQuantizationConfigArgs:
"""Configuration for online quantization.
class QuantSpec:
"""Quantization spec for one layer kind (linear or MoE).
Controls how ``OnlineQuantizationConfig`` is applied to a model.
At least one of ``global_scheme``, ``linear_scheme_override``, or
``moe_scheme_override`` must be set.
`None` on either side means the method class falls back to its own default
(typically inherited from the checkpoint, or unquantized for online).
"""
global_scheme: OnlineQuantScheme | None = None
"""Quantization scheme applied to every supported layer."""
weight: QuantKeyField = None
"""Weight quantization key, or a name from QUANT_KEY_NAMES."""
linear_scheme_override: OnlineQuantScheme | None = None
"""Quantization scheme override for ``LinearBase`` layers."""
activation: QuantKeyField = None
"""Activation quantization key, or a name from QUANT_KEY_NAMES."""
moe_scheme_override: OnlineQuantScheme | None = None
"""Quantization scheme override for ``FusedMoE`` layers."""
@config
class QuantizationConfigArgs:
"""User-facing quantization configuration.
See `docs/features/quantization/online.md` for the schema and shorthand
string forms accepted on `linear` and `moe`.
"""
linear: QuantSpec | None = None
"""Spec applied to ``LinearBase`` layers."""
moe: QuantSpec | None = None
"""Spec applied to ``FusedMoE`` layers."""
ignore: list[str] = Field(default_factory=list)
"""Layers to skip quantization for. Supports exact names and regex
patterns with ``re:`` prefix (e.g. ``re:.*attn.*``), consistent with
compressed_tensors layer skipping."""
"""Layers to skip quantization for."""
@field_validator(
"global_scheme", "linear_scheme_override", "moe_scheme_override", mode="before"
)
@field_validator("linear", "moe", mode="before")
@classmethod
def _coerce_scheme(
cls, v: str | OnlineQuantScheme | None
) -> OnlineQuantScheme | None:
if isinstance(v, str):
return OnlineQuantScheme(v)
return v
def _coerce_spec(cls, v: Any, info: ValidationInfo) -> Any:
if not isinstance(v, str):
return v
field_name = info.field_name
assert field_name is not None
if v in _ONLINE_SHORTHANDS:
spec = getattr(_ONLINE_SHORTHANDS[v], field_name)
if spec is None:
raise ValueError(
f"online shorthand {v!r} does not define a {field_name} spec"
)
return spec
return QuantSpec(weight=_coerce_quant_key(v))
def resolve_online_quant_config(
# CLI shorthands accepted by `--quantization`. Each desugars to a full
# QuantizationConfigArgs; activation overrides go through quantization_config.
_ONLINE_SHORTHANDS: dict[str, QuantizationConfigArgs] = {
"fp8_per_tensor": QuantizationConfigArgs(
linear=QuantSpec(weight=kFp8StaticTensorSym),
moe=QuantSpec(weight=kFp8StaticTensorSym),
),
"fp8_per_block": QuantizationConfigArgs(
linear=QuantSpec(weight=kFp8Static128BlockSym),
moe=QuantSpec(weight=kFp8Static128BlockSym),
),
"mxfp8": QuantizationConfigArgs(
linear=QuantSpec(weight=kMxfp8Dynamic),
moe=QuantSpec(weight=kMxfp8Dynamic),
),
# INT8 weight-only on MoE; linear stays unquantized (no `linear` field).
"int8_per_channel_weight_only": QuantizationConfigArgs(
moe=QuantSpec(weight=kInt8StaticChannelSym),
),
}
# Names accepted by `--quantization`; "online" means "use quantization_config".
ONLINE_QUANT_SHORTHAND_NAMES: tuple[str, ...] = (
*_ONLINE_SHORTHANDS.keys(),
"online",
)
def resolve_quantization_config(
quantization: str | None,
quantization_config: dict[str, Any] | OnlineQuantizationConfigArgs | None,
) -> OnlineQuantizationConfigArgs | None:
"""Resolve online quant scheme shorthand into a quantization config.
quantization_config: dict[str, Any] | QuantizationConfigArgs | None,
) -> QuantizationConfigArgs | None:
"""Resolve `--quantization` shorthand and `--quantization-config` into a
QuantizationConfigArgs.
If ``quantization`` is an online quant scheme (e.g. ``'fp8_per_tensor'``),
ensures ``quantization_config`` has a matching ``global_scheme`` and casts
it to :class:`OnlineQuantizationConfigArgs` if needed.
`quantization` is a CLI shorthand that desugars into a base config via
`_ONLINE_SHORTHANDS`. `quantization_config` is a dict or pre-built args
object. When both are given, fields explicitly set in `quantization_config`
take precedence over the shorthand.
"""
online_quant_values = {s.value for s in OnlineQuantScheme}
valid_quantization_values = online_quant_values | {"online"}
if quantization not in valid_quantization_values:
if quantization is not None and quantization not in ONLINE_QUANT_SHORTHAND_NAMES:
if quantization_config is not None:
raise ValueError(
f"quantization_config is only supported when quantization "
f"is one of {sorted(valid_quantization_values)}, "
f"quantization_config is only supported when quantization is "
f"one of {sorted(ONLINE_QUANT_SHORTHAND_NAMES)}, "
f"got quantization={quantization!r}"
)
return None
if quantization in online_quant_values:
scheme = OnlineQuantScheme(quantization)
base = _ONLINE_SHORTHANDS.get(quantization) if quantization else None
if quantization_config is None:
quantization_config = {
"global_scheme": scheme.value,
}
elif isinstance(quantization_config, OnlineQuantizationConfigArgs):
if quantization_config.global_scheme is None:
quantization_config.global_scheme = scheme
elif quantization_config.global_scheme != scheme:
raise ValueError(
f"quantization={quantization!r} conflicts with "
f"quantization_config.global_scheme="
f"{quantization_config.global_scheme.value!r}. "
f"These must match when both are specified."
)
elif isinstance(quantization_config, dict):
existing = quantization_config.get("global_scheme")
if existing is None:
quantization_config["global_scheme"] = scheme.value
else:
# Coerce to enum for comparison
existing_scheme = (
OnlineQuantScheme(existing)
if isinstance(existing, str)
else existing
)
if existing_scheme != scheme:
raise ValueError(
f"quantization={quantization!r} conflicts "
f"with quantization_config"
f"['global_scheme']={existing!r}. "
f"These must match when both are specified."
)
if quantization_config is None:
return base
# Cast dict to OnlineQuantizationConfigArgs
if isinstance(quantization_config, dict):
quantization_config = OnlineQuantizationConfigArgs(**quantization_config)
quantization_config = QuantizationConfigArgs(**quantization_config)
return quantization_config
if base is None:
return quantization_config
return QuantizationConfigArgs(
linear=quantization_config.linear or base.linear,
moe=quantization_config.moe or base.moe,
ignore=quantization_config.ignore or base.ignore,
)
+12 -4
View File
@@ -118,7 +118,7 @@ from vllm.v1.sample.logits_processor import LogitsProcessor
from vllm.version import __version__ as VLLM_VERSION
if TYPE_CHECKING:
from vllm.config.quantization import OnlineQuantizationConfigArgs
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.quantization import QuantizationMethods
from vllm.model_executor.model_loader import LoadFormats
from vllm.usage.usage_lib import UsageContext
@@ -527,7 +527,12 @@ class EngineArgs:
hf_overrides: HfOverrides = get_field(ModelConfig, "hf_overrides")
tokenizer_revision: str | None = ModelConfig.tokenizer_revision
quantization: QuantizationMethods | str | None = ModelConfig.quantization
quantization_config: "dict[str, Any] | OnlineQuantizationConfigArgs | None" = None
quantization_config: "dict[str, Any] | QuantizationConfigArgs | None" = None
"""User-facing quantization configuration. Carries per-layer-kind
QuantSpecs (linear, moe) and ignore patterns; see
:class:`QuantizationConfigArgs`. Auto-populated from the matching online
shorthand when `quantization` is one of the values in
`ONLINE_QUANT_SHORTHAND_NAMES`."""
allow_deprecated_quantization: bool = ModelConfig.allow_deprecated_quantization
enforce_eager: bool = ModelConfig.enforce_eager
disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
@@ -719,9 +724,9 @@ class EngineArgs:
if isinstance(self.ir_op_priority, dict):
self.ir_op_priority = IrOpPriorityConfig(**self.ir_op_priority)
from vllm.config.quantization import resolve_online_quant_config
from vllm.config.quantization import resolve_quantization_config
self.quantization_config = resolve_online_quant_config(
self.quantization_config = resolve_quantization_config(
self.quantization, self.quantization_config
)
@@ -785,6 +790,9 @@ class EngineArgs:
)
model_group.add_argument("--max-model-len", **model_kwargs["max_model_len"])
model_group.add_argument("--quantization", "-q", **model_kwargs["quantization"])
model_group.add_argument(
"--quantization-config", **model_kwargs["quantization_config"]
)
model_group.add_argument(
"--allow-deprecated-quantization",
**model_kwargs["allow_deprecated_quantization"],
+2 -4
View File
@@ -35,7 +35,7 @@ from vllm.config.model import (
TokenizerMode,
)
from vllm.config.quantization import (
OnlineQuantizationConfigArgs,
QuantizationConfigArgs,
)
from vllm.distributed.weight_transfer.base import (
WeightTransferInitRequest,
@@ -248,9 +248,7 @@ class LLM:
attention_config: dict[str, Any] | AttentionConfig | None = None,
kv_cache_memory_bytes: int | None = None,
compilation_config: int | dict[str, Any] | CompilationConfig | None = None,
quantization_config: dict[str, Any]
| OnlineQuantizationConfigArgs
| None = None,
quantization_config: dict[str, Any] | QuantizationConfigArgs | None = None,
logits_processors: list[str | type[LogitsProcessor]] | None = None,
**kwargs: Any,
) -> None:
@@ -1,13 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
from typing import TYPE_CHECKING, Union
from typing import TYPE_CHECKING, Literal, Union
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
from vllm.config import get_current_vllm_config
from vllm.config.kernel import MoEBackend
from vllm.config.quantization import QuantizationConfigArgs
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
@@ -207,25 +209,38 @@ def backend_to_kernel_cls(
raise ValueError(f"Unknown MXFP4 MoE backend: {backend.value}")
def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend:
"""Map user's moe_backend string to Mxfp4MoeBackend."""
mapping: dict[str, Mxfp4MoeBackend] = {
"deep_gemm": Mxfp4MoeBackend.DEEPGEMM_MXFP4,
"flashinfer_trtllm": Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
"flashinfer_trtllm_afp8": Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
"flashinfer_cutlass": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
"flashinfer_cutlass_afp8": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
"triton": Mxfp4MoeBackend.TRITON,
"triton_unfused": Mxfp4MoeBackend.TRITON_UNFUSED,
"humming": Mxfp4MoeBackend.HUMMING,
"marlin": Mxfp4MoeBackend.MARLIN,
"aiter": Mxfp4MoeBackend.AITER_MXFP4_BF16,
"aiter_mxfp4_fp8": Mxfp4MoeBackend.AITER_MXFP4_FP8,
"xpu": Mxfp4MoeBackend.XPU,
"emulation": Mxfp4MoeBackend.EMULATION,
def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
"""Map a moe_backend string to its candidate Mxfp4MoeBackends.
Vendor families return all activation variants; the caller picks one
via ``activation_key`` and ``is_supported_config``.
"""
mapping: dict[str, list[Mxfp4MoeBackend]] = {
"deep_gemm": [Mxfp4MoeBackend.DEEPGEMM_MXFP4],
"flashinfer_trtllm": [
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
],
"flashinfer_trtllm_afp8": [Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8],
"flashinfer_cutlass": [
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
],
"flashinfer_cutlass_afp8": [Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8],
"triton": [Mxfp4MoeBackend.TRITON],
"triton_unfused": [Mxfp4MoeBackend.TRITON_UNFUSED],
"humming": [Mxfp4MoeBackend.HUMMING],
"marlin": [Mxfp4MoeBackend.MARLIN],
"aiter": [
Mxfp4MoeBackend.AITER_MXFP4_BF16,
Mxfp4MoeBackend.AITER_MXFP4_FP8,
],
"aiter_mxfp4_fp8": [Mxfp4MoeBackend.AITER_MXFP4_FP8],
"xpu": [Mxfp4MoeBackend.XPU],
"emulation": [Mxfp4MoeBackend.EMULATION],
}
if backend := mapping.get(runner_backend):
return backend
if backends := mapping.get(runner_backend):
return backends
raise ValueError(
f"moe_backend='{runner_backend}' is not supported for MXFP4 MoE. "
f"Expected one of {list(mapping.keys())}."
@@ -233,16 +248,16 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend:
def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
"""
Get available backends in priority order based on platform and config.
Only includes BF16 backends. MXFP8 backends are selected via env vars.
"""
"""Available backends in priority order, BF16-act variant before
activation-quantized variant within each vendor family."""
_AVAILABLE_BACKENDS = [
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
Mxfp4MoeBackend.AITER_MXFP4_BF16,
Mxfp4MoeBackend.AITER_MXFP4_FP8,
Mxfp4MoeBackend.TRITON,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
# TRITON_UNFUSED has bug with MTP support
# TODO re-enable after kernel is fixed
# TRITON_UNFUSED
@@ -288,6 +303,79 @@ def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
return None # BF16 activation
def _user_moe_activation_override() -> QuantKey | None:
"""User's MoE activation override from quantization_config, or None."""
args = get_current_vllm_config().model_config.quantization_config
if not isinstance(args, QuantizationConfigArgs) or args.moe is None:
return None
return args.moe.activation
def _resolve_activation_key(
model_activation_key: QuantKey | None,
) -> QuantKey | None:
"""Combine the model-supplied activation key with the user override.
Raises on conflict (both set and disagreeing)."""
user_override = _user_moe_activation_override()
if user_override is None:
return model_activation_key
if model_activation_key is None or model_activation_key == user_override:
return user_override
raise ValueError(
f"checkpoint declares MoE activation={model_activation_key} but "
f"quantization_config.moe.activation={user_override}; remove the "
f"override or align it with the checkpoint."
)
def _make_log_backend(backend: Mxfp4MoeBackend) -> str:
return f"Using '{backend.value}' Mxfp4 MoE backend."
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
base = (
f"Mxfp4 MoE backend '{backend.value}' does not support the "
f"deployment configuration"
)
return f"{base} since {reason}." if reason else f"{base}."
def _return_or_raise(
backend: Mxfp4MoeBackend,
config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
scope: Literal["process", "global", "local"] = "local",
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
reason: str | None = None
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls, config, weight_key, activation_key, activation_format
)
if supported:
logger.info_once(_make_log_backend(backend), scope=scope)
return backend, k_cls
raise ValueError(_make_log_unsupported(backend, reason))
def _filter_by_activation(
backends: list[Mxfp4MoeBackend],
requested_activation_key: QuantKey | None,
) -> list[Mxfp4MoeBackend]:
"""Pick variants matching ``requested_activation_key``; without one,
prefer BF16 if the list has any, else keep the list as-is so explicit
non-BF16 picks (e.g. the ``_afp8`` aliases) still land."""
if requested_activation_key is not None:
return [
b
for b in backends
if _backend_activation_key(b) == requested_activation_key
]
bf16 = [b for b in backends if _backend_activation_key(b) is None]
return bf16 if bf16 else backends
def select_mxfp4_moe_backend(
config: FusedMoEConfig,
activation_key: QuantKey | None = None,
@@ -303,8 +391,7 @@ def select_mxfp4_moe_backend(
Note: Shape-specific fallbacks may still occur at runtime.
"""
# If activation_key is explicitly provided (e.g., W4A8), use it
requested_activation_key = activation_key
requested_activation_key = _resolve_activation_key(activation_key)
activation_format = (
mk.FusedMoEActivationFormat.BatchedExperts
@@ -312,67 +399,55 @@ def select_mxfp4_moe_backend(
else mk.FusedMoEActivationFormat.Standard
)
def _make_log_backend(backend: Mxfp4MoeBackend):
return f"Using '{backend.value}' Mxfp4 MoE backend."
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
if reason:
return (
f"Mxfp4 MoE backend '{backend.value}' does not support the "
f"deployment configuration since {reason}."
)
return (
f"Mxfp4 MoE backend '{backend.value}' does not support the "
"deployment configuration."
)
def _return_or_raise(
backend: Mxfp4MoeBackend,
config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
reason: str | None = None
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls, config, weight_key, activation_key, activation_format
)
if supported:
logger.info_once(_make_log_backend(backend))
return backend, k_cls
raise ValueError(_make_log_unsupported(backend, reason))
runner_backend = config.moe_backend
if runner_backend != "auto":
requested_backend = map_mxfp4_backend(runner_backend)
if (
activation_format == mk.FusedMoEActivationFormat.BatchedExperts
and requested_backend == Mxfp4MoeBackend.MARLIN
):
requested_backend = Mxfp4MoeBackend.BATCHED_MARLIN
# Use requested_activation_key if provided, otherwise use backend default
act_key = (
requested_activation_key
if requested_activation_key is not None
else _backend_activation_key(requested_backend)
)
return _return_or_raise(
requested_backend,
config,
kMxfp4Static,
act_key,
activation_format,
)
requested_backends = map_mxfp4_backend(runner_backend)
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
requested_backends = [
Mxfp4MoeBackend.BATCHED_MARLIN if b == Mxfp4MoeBackend.MARLIN else b
for b in requested_backends
]
candidates = _filter_by_activation(requested_backends, requested_activation_key)
if not candidates:
raise ValueError(
f"moe_backend={runner_backend!r} does not support "
f"activation={requested_activation_key}; supported variants: "
f"{[b.name for b in requested_backends]}"
)
last_error: Exception | None = None
for requested_backend in candidates:
act_key = (
requested_activation_key
if requested_activation_key is not None
else _backend_activation_key(requested_backend)
)
try:
return _return_or_raise(
requested_backend,
config,
kMxfp4Static,
act_key,
activation_format,
)
except ValueError as e:
last_error = e
assert last_error is not None
raise last_error
# Select kernels in order of backend.
AVAILABLE_BACKENDS = _get_priority_backends_for_gpt_oss()
AVAILABLE_BACKENDS = _filter_by_activation(
_get_priority_backends_for_gpt_oss(), requested_activation_key
)
# Handle explicit FlashInfer MXFP4 BF16 configuration.
if envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16"):
if not envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16:
AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16)
AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16)
for _b in (
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
):
if _b in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(_b)
else:
if current_platform.is_device_capability(90):
return _return_or_raise(
@@ -481,54 +556,30 @@ def select_deepseek_v4_mxfp4_moe_backend(
else mk.FusedMoEActivationFormat.Standard
)
def _make_log_backend(backend: Mxfp4MoeBackend):
return f"Using '{backend.value}' Mxfp4 MoE backend."
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
if reason:
return (
f"Mxfp4 MoE backend '{backend.value}' does not support the "
f"deployment configuration since {reason}."
)
return (
f"Mxfp4 MoE backend '{backend.value}' does not support the "
"deployment configuration."
)
def _return_or_raise(
backend: Mxfp4MoeBackend,
config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
reason: str | None = None
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls, config, weight_key, activation_key, activation_format
)
if supported:
logger.info_once(_make_log_backend(backend), scope="local")
return backend, k_cls
raise ValueError(_make_log_unsupported(backend, reason))
# Honor explicit moe_backend (e.g. "marlin", "triton_unfused") before
# falling back to the auto priority list.
runner_backend = config.moe_backend
if runner_backend != "auto":
requested_backend = map_mxfp4_backend(runner_backend)
if (
activation_format == mk.FusedMoEActivationFormat.BatchedExperts
and requested_backend == Mxfp4MoeBackend.MARLIN
):
requested_backend = Mxfp4MoeBackend.BATCHED_MARLIN
return _return_or_raise(
requested_backend,
config,
kMxfp4Static,
_backend_activation_key(requested_backend),
activation_format,
)
requested_backends = map_mxfp4_backend(runner_backend)
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
requested_backends = [
Mxfp4MoeBackend.BATCHED_MARLIN if b == Mxfp4MoeBackend.MARLIN else b
for b in requested_backends
]
last_error: Exception | None = None
for requested_backend in requested_backends:
try:
return _return_or_raise(
requested_backend,
config,
kMxfp4Static,
_backend_activation_key(requested_backend),
activation_format,
)
except ValueError as e:
last_error = e
assert last_error is not None
raise last_error
# DeepSeek-V4 on ROCm is more accurate with the unfused Triton MXFP4 path
# than the default AITER path. Prefer Triton-unfused for this routing mode,
@@ -35,10 +35,9 @@ QuantizationMethods = Literal[
"deepseek_v4_fp8",
"cpu_awq",
"online",
# Below are values of the OnlineQuantScheme enum, specified as strings to
# avoid circular import issues. This is here to provide a shortcut where
# the user can specify "LLM(..., quantization='fp8_per_tensor')" as
# shorthand for creating a more complicated online quant config object
# Below are online quant shorthand names (see vllm.config.quantization).
# Listed here as strings to avoid a circular import; kept in sync with
# _ONLINE_SHORTHANDS by the assertion in get_quantization_config().
"fp8_per_tensor",
"fp8_per_block",
"int8_per_channel_weight_only",
@@ -111,7 +110,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
raise ValueError(f"Invalid quantization method: {quantization}")
# lazy import to avoid triggering `torch.compile` too early
from vllm.config.quantization import OnlineQuantScheme
from vllm.config.quantization import _ONLINE_SHORTHANDS
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.models.deepseek_v4 import DeepseekV4FP8Config
@@ -171,16 +170,15 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
"online": OnlineQuantizationConfig,
}
# Below are values of the OnlineQuantScheme enum. This is here to provide
# a shortcut where the user can specify
# "LLM(..., quantization='fp8_per_tensor')" as shorthand for creating a
# more complicated online quant config object
for scheme in OnlineQuantScheme:
assert scheme.value not in method_to_config, (
f"Online quant scheme {scheme.value!r} conflicts with an "
# Register online shorthands as quantization methods so the user can
# specify "LLM(..., quantization='fp8_per_tensor')" as shorthand for
# creating a more complicated online quant config object.
for shorthand in _ONLINE_SHORTHANDS:
assert shorthand not in method_to_config, (
f"Online quant shorthand {shorthand!r} conflicts with an "
f"existing quantization method"
)
method_to_config[scheme.value] = OnlineQuantizationConfig
method_to_config[shorthand] = OnlineQuantizationConfig
# Update the `method_to_config` with customized quantization methods.
method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG)
@@ -5,10 +5,7 @@ from typing import Any
import torch
from vllm.config.quantization import (
OnlineQuantizationConfigArgs,
OnlineQuantScheme,
)
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
RoutedExperts,
@@ -41,31 +38,50 @@ from vllm.model_executor.layers.quantization.online.mxfp8 import (
Mxfp8OnlineLinearMethod,
Mxfp8OnlineMoEMethod,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
kInt8StaticChannelSym,
kMxfp8Dynamic,
)
logger = init_logger(__name__)
# Online dispatch tables, keyed by the QuantSpec.weight QuantKey. The
# corresponding method class handles the activation choice via its
# `supported_activation_quant` set.
_ONLINE_LINEAR_METHODS: dict[QuantKey, type] = {
kFp8StaticTensorSym: Fp8PerTensorOnlineLinearMethod,
kFp8Static128BlockSym: Fp8PerBlockOnlineLinearMethod,
kMxfp8Dynamic: Mxfp8OnlineLinearMethod,
}
_ONLINE_MOE_METHODS: dict[QuantKey, type] = {
kFp8StaticTensorSym: Fp8PerTensorOnlineMoEMethod,
kFp8Static128BlockSym: Fp8PerBlockOnlineMoEMethod,
kMxfp8Dynamic: Mxfp8OnlineMoEMethod,
kInt8StaticChannelSym: Int8OnlineMoEMethod,
}
class OnlineQuantizationConfig(QuantizationConfig):
"""Model-level config class for online quantization (quantize fp16/bf16 weights
"""Model-level config for online quantization (quantize fp16/bf16 weights
during model loading, without requiring a pre-quantized checkpoint)."""
def __init__(
self,
args: OnlineQuantizationConfigArgs,
args: QuantizationConfigArgs,
) -> None:
super().__init__()
if (
args.global_scheme is None
and args.linear_scheme_override is None
and args.moe_scheme_override is None
):
if args.linear is None and args.moe is None:
raise ValueError(
"OnlineQuantizationConfig requires at least one of "
"global_scheme, linear_scheme_override, or "
"moe_scheme_override to be set."
"quantization_config.linear or quantization_config.moe "
"to be set."
)
self.args = args
self.quant_scheme = args.global_scheme
self.ignored_layers: list[str] = args.ignore
@classmethod
@@ -94,6 +110,33 @@ class OnlineQuantizationConfig(QuantizationConfig):
"quantization='fp8_per_tensor'/'fp8_per_block' instead."
)
def _dispatch(
self,
spec: QuantSpec | None,
table: dict[QuantKey, type],
layer: torch.nn.Module,
) -> "QuantizeMethodBase | None":
if spec is None or spec.weight is None:
return None
cls = table.get(spec.weight)
if cls is None:
raise ValueError(
f"online quantization for {type(layer).__name__} with "
f"weight={spec.weight} is not supported; supported weight "
f"keys: {sorted(str(k) for k in table)}"
)
# Online method classes pick their own activation format internally.
# Per-class activation overrides are not yet wired through; reject
# explicit overrides until the relevant method class opts in.
if spec.activation is not None:
raise ValueError(
f"activation override (activation={spec.activation}) is not "
f"yet supported for online {cls.__name__}"
)
if isinstance(layer, RoutedExperts):
return cls(layer=layer)
return cls()
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
@@ -104,20 +147,8 @@ class OnlineQuantizationConfig(QuantizationConfig):
fused_mapping=self.packed_modules_mapping,
):
return UnquantizedLinearMethod()
linear_scheme = self.args.linear_scheme_override or self.args.global_scheme
if linear_scheme == OnlineQuantScheme.INT8_PER_CHANNEL_WEIGHT_ONLY:
logger.warning_once(
"INT8 online quantization only quantizes MoE expert "
"weights. linear layers remain in full precision."
)
return UnquantizedLinearMethod()
elif linear_scheme == OnlineQuantScheme.FP8_PER_BLOCK:
return Fp8PerBlockOnlineLinearMethod()
elif linear_scheme == OnlineQuantScheme.MXFP8:
return Mxfp8OnlineLinearMethod()
else:
return Fp8PerTensorOnlineLinearMethod()
method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer)
return method if method is not None else UnquantizedLinearMethod()
elif isinstance(layer, RoutedExperts):
if should_ignore_layer(
prefix,
@@ -125,14 +156,10 @@ class OnlineQuantizationConfig(QuantizationConfig):
fused_mapping=self.packed_modules_mapping,
):
return UnquantizedFusedMoEMethod(layer.moe_config)
moe_scheme = self.args.moe_scheme_override or self.args.global_scheme
if moe_scheme == OnlineQuantScheme.INT8_PER_CHANNEL_WEIGHT_ONLY:
return Int8OnlineMoEMethod(layer=layer)
elif moe_scheme == OnlineQuantScheme.FP8_PER_BLOCK:
return Fp8PerBlockOnlineMoEMethod(layer=layer)
elif moe_scheme == OnlineQuantScheme.MXFP8:
return Mxfp8OnlineMoEMethod(layer=layer)
else:
return Fp8PerTensorOnlineMoEMethod(layer=layer)
method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer)
return (
method
if method is not None
else UnquantizedFusedMoEMethod(layer.moe_config)
)
return None
@@ -300,12 +300,10 @@ def get_quant_config(
)
if hf_quant_config is not None:
if model_config.quantization_config is not None:
raise ValueError(
"Setting `quantization_config` for online "
"quantization when the model checkpoint already "
"has a `quantization_config` is not supported"
)
# `model_config.quantization_config` may be set alongside a checkpoint
# quant config: the checkpoint determines `quant_cls`, and the user's
# QuantizationConfigArgs is consulted by individual quant methods
# (e.g. for activation overrides via the MXFP4 oracle).
# For modelopt_mixed, config.json's quantization_config may or may
# not contain the per-layer quantized_layers map. Newer checkpoints
@@ -330,12 +328,6 @@ def get_quant_config(
quantization_config_file = hf_overrides.get("quantization_config_file", None)
if quantization_config_file is not None:
if hasattr(quant_cls, "from_config_file"):
if model_config.quantization_config is not None:
raise ValueError(
"Setting `quantization_config` for online "
"quantization when the model checkpoint already "
"has a `quantization_config` is not supported"
)
return quant_cls.from_config_file(quantization_config_file)
else:
raise NotImplementedError(
@@ -346,12 +338,6 @@ def get_quant_config(
quantization_config_json = hf_overrides.get("quantization_config_dict_json", None)
if quantization_config_json is not None:
if hasattr(quant_cls, "from_config_dict_json"):
if model_config.quantization_config is not None:
raise ValueError(
"Setting `quantization_config` for online "
"quantization when the model checkpoint already "
"has a `quantization_config` is not supported"
)
return quant_cls.from_config_dict_json(quantization_config_json)
else:
raise NotImplementedError(
@@ -360,17 +346,15 @@ def get_quant_config(
f"{quant_cls}"
)
# Online quantization doesn't read from checkpoint configs — it quantizes
# Online quantization doesn't read from checkpoint configs - it quantizes
# fp16/bf16 weights on the fly during loading.
if model_config.quantization_config is not None:
from vllm.config.quantization import OnlineQuantizationConfigArgs
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
assert isinstance(
model_config.quantization_config, OnlineQuantizationConfigArgs
)
assert isinstance(model_config.quantization_config, QuantizationConfigArgs)
return OnlineQuantizationConfig(args=model_config.quantization_config)
# Inflight BNB quantization