[Quantization] Rework quantization_config to use QuantKey and allow for activation override (#41566)
This commit is contained in:
@@ -42,18 +42,59 @@ vllm serve meta-llama/Llama-3.1-8B --quantization mxfp8
|
||||
|
||||
For fine-grained control, use a `quantization_config` dictionary.
|
||||
|
||||
### Schema
|
||||
|
||||
```yaml
|
||||
quantization_config:
|
||||
linear:
|
||||
weight: <name> # see QUANT_KEY_NAMES in vllm/config/quantization.py
|
||||
activation: <name>
|
||||
moe:
|
||||
weight: <name>
|
||||
activation: <name>
|
||||
ignore: [<layer-name-or-regex>, ...]
|
||||
```
|
||||
|
||||
`linear` and `moe` accept a full `{weight, activation}` dict, or a bare
|
||||
string. A string resolves first against the `--quantization` shorthands
|
||||
(taking the matching layer-kind slot), then against `QUANT_KEY_NAMES` as a
|
||||
weight name. Unset fields fall back to the `--quantization` shorthand's
|
||||
defaults, or for already-quantized checkpoints to whatever the checkpoint
|
||||
declares.
|
||||
|
||||
The CLI accepts the same shape as JSON or as dotted keys:
|
||||
|
||||
```bash
|
||||
vllm serve <model> --quantization-config '{"moe":{"activation":"mxfp8"}}'
|
||||
vllm serve <model> --quantization-config.moe.activation mxfp8
|
||||
```
|
||||
|
||||
### Activation overrides on already-quantized checkpoints
|
||||
|
||||
For checkpoint-quantized models, `quantization_config` lets you pick an
|
||||
activation format independently of the baked-in weights. The supported
|
||||
overrides are checkpoint-specific; today this is wired up for MXFP4 MoE
|
||||
checkpoints (gpt-oss) where you can opt into FP8 activations:
|
||||
|
||||
```bash
|
||||
vllm serve openai/gpt-oss-20b --quantization-config.moe.activation mxfp8
|
||||
```
|
||||
|
||||
Combine with `--moe-backend` to pin a specific kernel family.
|
||||
|
||||
### Separate Schemes for Dense and MoE Layers
|
||||
|
||||
You can apply different quantization schemes to dense linear layers and MoE expert layers:
|
||||
You can apply different quantization schemes to dense linear layers and MoE expert layers via the `linear` and `moe` fields. Each accepts either a full spec dict, or a bare string naming an online shorthand (e.g. `"fp8_per_block"`) or weight format (e.g. `"fp8_per_block_static"`); fields not set fall back to the shorthand defaults.
|
||||
|
||||
```python
|
||||
from vllm import LLM
|
||||
|
||||
# Linear: per-block FP8; MoE: per-tensor FP8 (inherited from the shorthand)
|
||||
llm = LLM(
|
||||
"ibm-granite/granite-3.0-1b-a400m-base",
|
||||
quantization="fp8_per_tensor",
|
||||
quantization_config={
|
||||
"linear_scheme_override": "fp8_per_block",
|
||||
"linear": "fp8_per_block",
|
||||
},
|
||||
)
|
||||
```
|
||||
@@ -63,11 +104,12 @@ Or,
|
||||
```python
|
||||
from vllm import LLM
|
||||
|
||||
# Linear: per-tensor FP8 (inherited); MoE: per-block FP8
|
||||
llm = LLM(
|
||||
"ibm-granite/granite-3.0-1b-a400m-base",
|
||||
quantization="fp8_per_tensor",
|
||||
quantization_config={
|
||||
"moe_scheme_override": "fp8_per_block",
|
||||
"moe": "fp8_per_block",
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
@@ -103,13 +103,6 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
|
||||
if not supports_trtllm_attention():
|
||||
matches = matches._replace(attn_quant_fusion=0)
|
||||
|
||||
# TODO: remove this after finishing migration from envs to model kwargs
|
||||
if model_name == "openai/gpt-oss-20b":
|
||||
from .common import is_blackwell
|
||||
|
||||
if is_blackwell():
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "1")
|
||||
|
||||
# Disable, compile cache to make sure custom passes run.
|
||||
# Otherwise, we can't verify fusion happened through the logs.
|
||||
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
|
||||
|
||||
@@ -214,4 +214,9 @@ gpt_oss_20b = ModelFusionInfo(
|
||||
sequence_parallel=n_layers * 2 + 1,
|
||||
async_tp=n_layers * 2,
|
||||
),
|
||||
model_kwargs=(
|
||||
{"quantization_config": {"moe": {"activation": "mxfp8"}}}
|
||||
if is_blackwell()
|
||||
else {}
|
||||
),
|
||||
)
|
||||
|
||||
@@ -3,6 +3,4 @@
|
||||
model_name: "openai/gpt-oss-20b"
|
||||
metric_threshold: 0.568
|
||||
reasoning_effort: "low"
|
||||
server_args: "--tensor-parallel-size 2"
|
||||
env:
|
||||
VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS: "1"
|
||||
server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_cutlass --quantization-config.moe.activation mxfp8"
|
||||
|
||||
@@ -3,6 +3,4 @@
|
||||
model_name: "openai/gpt-oss-20b"
|
||||
metric_threshold: 0.568
|
||||
reasoning_effort: "low"
|
||||
server_args: "--tensor-parallel-size 2"
|
||||
env:
|
||||
VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: "1"
|
||||
server_args: "--tensor-parallel-size 2 --quantization-config.moe.activation mxfp8"
|
||||
|
||||
@@ -190,13 +190,27 @@ def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
|
||||
def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", "1")
|
||||
can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT)
|
||||
can_initialize(
|
||||
"openai/gpt-oss-20b",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=[
|
||||
"--moe-backend",
|
||||
"flashinfer_cutlass",
|
||||
"--quantization-config.moe.activation",
|
||||
"mxfp8",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_gptoss_mxfp4mxfp8_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "1")
|
||||
can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT)
|
||||
can_initialize(
|
||||
"openai/gpt-oss-20b",
|
||||
hf_overrides=HF_OVERRIDE_TEXT,
|
||||
extra_args=[
|
||||
"--quantization-config.moe.activation",
|
||||
"mxfp8",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_gptoss_eager(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
@@ -40,13 +40,12 @@ from vllm.platforms import current_platform
|
||||
Fp8PerBlockOnlineLinearMethod,
|
||||
Fp8PerBlockOnlineMoEMethod,
|
||||
),
|
||||
# quantization='online with linear_scheme_override and
|
||||
# moe_scheme_override
|
||||
# quantization='online' with per-layer-kind overrides
|
||||
(
|
||||
"online",
|
||||
{
|
||||
"linear_scheme_override": "fp8_per_block",
|
||||
"moe_scheme_override": "fp8_per_tensor",
|
||||
"linear": "fp8_per_block",
|
||||
"moe": "fp8_per_tensor",
|
||||
},
|
||||
Fp8PerBlockOnlineLinearMethod,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for QuantizationConfigArgs parsing."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.quantization import (
|
||||
QUANT_KEY_NAMES,
|
||||
QuantizationConfigArgs,
|
||||
QuantSpec,
|
||||
resolve_quantization_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp8Dynamic,
|
||||
)
|
||||
|
||||
# ---- QuantSpec ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_quant_spec_resolves_string_to_quant_key():
|
||||
spec = QuantSpec(weight="mxfp8", activation="fp8_per_token")
|
||||
assert spec.weight == kMxfp8Dynamic
|
||||
assert spec.activation == kFp8DynamicTokenSym
|
||||
|
||||
|
||||
def test_quant_spec_accepts_quant_key_directly():
|
||||
spec = QuantSpec(weight=kFp8StaticTensorSym)
|
||||
assert spec.weight is kFp8StaticTensorSym
|
||||
assert spec.activation is None
|
||||
|
||||
|
||||
def test_quant_spec_rejects_unknown_name():
|
||||
with pytest.raises(ValueError, match="unknown quantization name"):
|
||||
QuantSpec(weight="not_a_real_format")
|
||||
|
||||
|
||||
# ---- QuantizationConfigArgs string shorthand on linear/moe ----------------
|
||||
|
||||
|
||||
def test_args_linear_string_resolves_via_quant_key_names():
|
||||
# A bare QUANT_KEY_NAMES entry desugars to QuantSpec(weight=<key>).
|
||||
args = QuantizationConfigArgs(linear="fp8_per_block_static")
|
||||
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
assert args.moe is None
|
||||
|
||||
|
||||
def test_args_moe_string_resolves_via_online_shorthand():
|
||||
# An online-shorthand name pulls the matching slot from _ONLINE_SHORTHANDS
|
||||
# (so `linear: "fp8_per_block"` and `moe: "fp8_per_block"` produce the
|
||||
# same per-layer-kind spec the `--quantization fp8_per_block` shorthand
|
||||
# would).
|
||||
args = QuantizationConfigArgs(moe="fp8_per_block")
|
||||
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
|
||||
|
||||
def test_args_string_shorthand_missing_slot_raises():
|
||||
# int8_per_channel_weight_only sets only `moe`; using it on `linear`
|
||||
# has no defined spec and should raise rather than silently no-op.
|
||||
with pytest.raises(ValueError, match="does not define a linear spec"):
|
||||
QuantizationConfigArgs(linear="int8_per_channel_weight_only")
|
||||
|
||||
|
||||
def test_args_accepts_dict_form():
|
||||
args = QuantizationConfigArgs(moe={"activation": "mxfp8"})
|
||||
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
|
||||
|
||||
|
||||
# ---- resolve_quantization_config -----------------------------------------
|
||||
|
||||
|
||||
def test_resolve_shorthand_only_populates_both_slots():
|
||||
args = resolve_quantization_config("fp8_per_block", None)
|
||||
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
|
||||
|
||||
def test_resolve_int8_shorthand_leaves_linear_unset():
|
||||
# int8_per_channel_weight_only is MoE-only; linear stays None so that
|
||||
# OnlineQuantizationConfig leaves Linear layers in full precision.
|
||||
args = resolve_quantization_config("int8_per_channel_weight_only", None)
|
||||
assert args.linear is None
|
||||
assert args.moe == QuantSpec(weight=kInt8StaticChannelSym)
|
||||
|
||||
|
||||
def test_resolve_quantization_config_only():
|
||||
# When only `quantization_config` is given (e.g. for an already-quantized
|
||||
# checkpoint that needs an activation override), it's returned as-is.
|
||||
args = resolve_quantization_config(None, {"moe": {"activation": "mxfp8"}})
|
||||
assert args.linear is None
|
||||
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
|
||||
|
||||
|
||||
def test_resolve_merges_explicit_over_shorthand():
|
||||
# Explicit linear in quantization_config wins; moe falls back to the
|
||||
# shorthand's slot.
|
||||
args = resolve_quantization_config(
|
||||
"fp8_per_tensor",
|
||||
{"linear": "fp8_per_block"},
|
||||
)
|
||||
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
assert args.moe == QuantSpec(weight=kFp8StaticTensorSym)
|
||||
|
||||
|
||||
def test_resolve_rejects_quantization_config_with_non_shorthand_quant():
|
||||
# If --quantization names something other than an online shorthand,
|
||||
# quantization_config is not allowed via this path (checkpoint quant
|
||||
# paths read it directly off ModelConfig instead).
|
||||
with pytest.raises(ValueError, match="quantization_config is only supported"):
|
||||
resolve_quantization_config("gptq", {"linear": "fp8_per_block"})
|
||||
|
||||
|
||||
# ---- QUANT_KEY_NAMES coverage --------------------------------------------
|
||||
|
||||
|
||||
def test_quant_key_names_round_trip():
|
||||
# Every advertised name should round-trip through QuantSpec without error
|
||||
# and produce the same QuantKey it maps to.
|
||||
for name, expected in QUANT_KEY_NAMES.items():
|
||||
assert QuantSpec(weight=name).weight == expected, name
|
||||
assert QuantSpec(activation=name).activation == expected, name
|
||||
|
||||
|
||||
def test_static_block_weight_paired_with_dynamic_block_activation():
|
||||
# The block-FP8 shorthand pair: 128x128 static weights + 1x128 dynamic
|
||||
# activations. Pinning this so renames in QUANT_KEY_NAMES don't quietly
|
||||
# rewire the kernel dispatch.
|
||||
spec = QuantSpec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
|
||||
assert spec.weight == kFp8Static128BlockSym
|
||||
assert spec.activation == kFp8Dynamic128Sym
|
||||
@@ -21,7 +21,7 @@ from vllm.config.multimodal import (
|
||||
MultiModalConfig,
|
||||
)
|
||||
from vllm.config.pooler import PoolerConfig
|
||||
from vllm.config.quantization import OnlineQuantizationConfigArgs
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.config.scheduler import RunnerType
|
||||
from vllm.config.utils import config, getattr_iter
|
||||
from vllm.logger import init_logger
|
||||
@@ -206,10 +206,11 @@ class ModelConfig:
|
||||
`quantization_config` attribute in the model config file. If that is
|
||||
`None`, we assume the model weights are not quantized and use `dtype` to
|
||||
determine the data type of the weights."""
|
||||
quantization_config: dict[str, Any] | OnlineQuantizationConfigArgs | None = None
|
||||
"""Arguments for online quantization.
|
||||
Auto-created when `quantization` equals to one of the string values of
|
||||
the `OnlineQuantScheme` enum."""
|
||||
quantization_config: dict[str, Any] | QuantizationConfigArgs | None = None
|
||||
"""User-facing quantization configuration. Carries per-layer-kind specs
|
||||
(linear, moe) and ignore patterns; see :class:`QuantizationConfigArgs`.
|
||||
Auto-populated from the matching online shorthand when `quantization` is
|
||||
one of the values in `ONLINE_QUANT_SHORTHAND_NAMES`."""
|
||||
allow_deprecated_quantization: bool = False
|
||||
"""Whether to allow deprecated quantization methods."""
|
||||
enforce_eager: bool = False
|
||||
|
||||
+138
-89
@@ -1,126 +1,175 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
from typing import Annotated, Any
|
||||
|
||||
from pydantic import Field, field_validator
|
||||
from pydantic import Field, GetPydanticSchema, ValidationInfo, field_validator
|
||||
from pydantic_core import core_schema
|
||||
|
||||
from vllm.config.utils import config
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp8Dynamic,
|
||||
)
|
||||
|
||||
# User-facing names addressable from quantization_config.
|
||||
QUANT_KEY_NAMES: dict[str, QuantKey] = {
|
||||
"fp8_per_tensor_static": kFp8StaticTensorSym,
|
||||
"fp8_per_tensor_dynamic": kFp8DynamicTensorSym,
|
||||
"fp8_per_token": kFp8DynamicTokenSym,
|
||||
"fp8_per_block_static": kFp8Static128BlockSym,
|
||||
"fp8_per_block_dynamic": kFp8Dynamic128Sym,
|
||||
"mxfp8": kMxfp8Dynamic,
|
||||
"mxfp4": kMxfp4Dynamic,
|
||||
"int8_per_channel_static": kInt8StaticChannelSym,
|
||||
}
|
||||
|
||||
|
||||
class OnlineQuantScheme(Enum):
|
||||
"""Supported online quantization schemes."""
|
||||
def _coerce_quant_key(v: Any) -> QuantKey | None:
|
||||
if v is None or isinstance(v, QuantKey):
|
||||
return v
|
||||
if not isinstance(v, str):
|
||||
raise TypeError(f"expected str or QuantKey, got {type(v).__name__}")
|
||||
try:
|
||||
return QUANT_KEY_NAMES[v]
|
||||
except KeyError:
|
||||
raise ValueError(
|
||||
f"unknown quantization name {v!r}; "
|
||||
f"expected one of {sorted(QUANT_KEY_NAMES)}"
|
||||
) from None
|
||||
|
||||
# fp8, weights and activations scaled per-tensor
|
||||
FP8_PER_TENSOR = "fp8_per_tensor"
|
||||
|
||||
# fp8, activations scaled in blocks of 1x128 elements, weights scaled in
|
||||
# blocks of 128x128 elements (popularized by DeepSeek)
|
||||
FP8_PER_BLOCK = "fp8_per_block"
|
||||
|
||||
# int8, weight-only per-channel quantization for MoE expert weights.
|
||||
# Linear layers remain unquantized.
|
||||
INT8_PER_CHANNEL_WEIGHT_ONLY = "int8_per_channel_weight_only"
|
||||
|
||||
# mxfp8, weights scaled in blocks of 1x32 elements (microscaling FP8)
|
||||
MXFP8 = "mxfp8"
|
||||
# Stop pydantic from introspecting QuantKey: it transitively contains a
|
||||
# NamedTuple with `ClassVar[GroupShape]` declarations that pydantic refuses.
|
||||
QuantKeyField = Annotated[
|
||||
QuantKey | None,
|
||||
GetPydanticSchema(
|
||||
lambda _src, _handler: core_schema.no_info_plain_validator_function(
|
||||
_coerce_quant_key
|
||||
)
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@config
|
||||
class OnlineQuantizationConfigArgs:
|
||||
"""Configuration for online quantization.
|
||||
class QuantSpec:
|
||||
"""Quantization spec for one layer kind (linear or MoE).
|
||||
|
||||
Controls how ``OnlineQuantizationConfig`` is applied to a model.
|
||||
At least one of ``global_scheme``, ``linear_scheme_override``, or
|
||||
``moe_scheme_override`` must be set.
|
||||
`None` on either side means the method class falls back to its own default
|
||||
(typically inherited from the checkpoint, or unquantized for online).
|
||||
"""
|
||||
|
||||
global_scheme: OnlineQuantScheme | None = None
|
||||
"""Quantization scheme applied to every supported layer."""
|
||||
weight: QuantKeyField = None
|
||||
"""Weight quantization key, or a name from QUANT_KEY_NAMES."""
|
||||
|
||||
linear_scheme_override: OnlineQuantScheme | None = None
|
||||
"""Quantization scheme override for ``LinearBase`` layers."""
|
||||
activation: QuantKeyField = None
|
||||
"""Activation quantization key, or a name from QUANT_KEY_NAMES."""
|
||||
|
||||
moe_scheme_override: OnlineQuantScheme | None = None
|
||||
"""Quantization scheme override for ``FusedMoE`` layers."""
|
||||
|
||||
@config
|
||||
class QuantizationConfigArgs:
|
||||
"""User-facing quantization configuration.
|
||||
|
||||
See `docs/features/quantization/online.md` for the schema and shorthand
|
||||
string forms accepted on `linear` and `moe`.
|
||||
"""
|
||||
|
||||
linear: QuantSpec | None = None
|
||||
"""Spec applied to ``LinearBase`` layers."""
|
||||
|
||||
moe: QuantSpec | None = None
|
||||
"""Spec applied to ``FusedMoE`` layers."""
|
||||
|
||||
ignore: list[str] = Field(default_factory=list)
|
||||
"""Layers to skip quantization for. Supports exact names and regex
|
||||
patterns with ``re:`` prefix (e.g. ``re:.*attn.*``), consistent with
|
||||
compressed_tensors layer skipping."""
|
||||
"""Layers to skip quantization for."""
|
||||
|
||||
@field_validator(
|
||||
"global_scheme", "linear_scheme_override", "moe_scheme_override", mode="before"
|
||||
)
|
||||
@field_validator("linear", "moe", mode="before")
|
||||
@classmethod
|
||||
def _coerce_scheme(
|
||||
cls, v: str | OnlineQuantScheme | None
|
||||
) -> OnlineQuantScheme | None:
|
||||
if isinstance(v, str):
|
||||
return OnlineQuantScheme(v)
|
||||
return v
|
||||
def _coerce_spec(cls, v: Any, info: ValidationInfo) -> Any:
|
||||
if not isinstance(v, str):
|
||||
return v
|
||||
field_name = info.field_name
|
||||
assert field_name is not None
|
||||
if v in _ONLINE_SHORTHANDS:
|
||||
spec = getattr(_ONLINE_SHORTHANDS[v], field_name)
|
||||
if spec is None:
|
||||
raise ValueError(
|
||||
f"online shorthand {v!r} does not define a {field_name} spec"
|
||||
)
|
||||
return spec
|
||||
return QuantSpec(weight=_coerce_quant_key(v))
|
||||
|
||||
|
||||
def resolve_online_quant_config(
|
||||
# CLI shorthands accepted by `--quantization`. Each desugars to a full
|
||||
# QuantizationConfigArgs; activation overrides go through quantization_config.
|
||||
_ONLINE_SHORTHANDS: dict[str, QuantizationConfigArgs] = {
|
||||
"fp8_per_tensor": QuantizationConfigArgs(
|
||||
linear=QuantSpec(weight=kFp8StaticTensorSym),
|
||||
moe=QuantSpec(weight=kFp8StaticTensorSym),
|
||||
),
|
||||
"fp8_per_block": QuantizationConfigArgs(
|
||||
linear=QuantSpec(weight=kFp8Static128BlockSym),
|
||||
moe=QuantSpec(weight=kFp8Static128BlockSym),
|
||||
),
|
||||
"mxfp8": QuantizationConfigArgs(
|
||||
linear=QuantSpec(weight=kMxfp8Dynamic),
|
||||
moe=QuantSpec(weight=kMxfp8Dynamic),
|
||||
),
|
||||
# INT8 weight-only on MoE; linear stays unquantized (no `linear` field).
|
||||
"int8_per_channel_weight_only": QuantizationConfigArgs(
|
||||
moe=QuantSpec(weight=kInt8StaticChannelSym),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# Names accepted by `--quantization`; "online" means "use quantization_config".
|
||||
ONLINE_QUANT_SHORTHAND_NAMES: tuple[str, ...] = (
|
||||
*_ONLINE_SHORTHANDS.keys(),
|
||||
"online",
|
||||
)
|
||||
|
||||
|
||||
def resolve_quantization_config(
|
||||
quantization: str | None,
|
||||
quantization_config: dict[str, Any] | OnlineQuantizationConfigArgs | None,
|
||||
) -> OnlineQuantizationConfigArgs | None:
|
||||
"""Resolve online quant scheme shorthand into a quantization config.
|
||||
quantization_config: dict[str, Any] | QuantizationConfigArgs | None,
|
||||
) -> QuantizationConfigArgs | None:
|
||||
"""Resolve `--quantization` shorthand and `--quantization-config` into a
|
||||
QuantizationConfigArgs.
|
||||
|
||||
If ``quantization`` is an online quant scheme (e.g. ``'fp8_per_tensor'``),
|
||||
ensures ``quantization_config`` has a matching ``global_scheme`` and casts
|
||||
it to :class:`OnlineQuantizationConfigArgs` if needed.
|
||||
`quantization` is a CLI shorthand that desugars into a base config via
|
||||
`_ONLINE_SHORTHANDS`. `quantization_config` is a dict or pre-built args
|
||||
object. When both are given, fields explicitly set in `quantization_config`
|
||||
take precedence over the shorthand.
|
||||
"""
|
||||
online_quant_values = {s.value for s in OnlineQuantScheme}
|
||||
valid_quantization_values = online_quant_values | {"online"}
|
||||
if quantization not in valid_quantization_values:
|
||||
if quantization is not None and quantization not in ONLINE_QUANT_SHORTHAND_NAMES:
|
||||
if quantization_config is not None:
|
||||
raise ValueError(
|
||||
f"quantization_config is only supported when quantization "
|
||||
f"is one of {sorted(valid_quantization_values)}, "
|
||||
f"quantization_config is only supported when quantization is "
|
||||
f"one of {sorted(ONLINE_QUANT_SHORTHAND_NAMES)}, "
|
||||
f"got quantization={quantization!r}"
|
||||
)
|
||||
return None
|
||||
|
||||
if quantization in online_quant_values:
|
||||
scheme = OnlineQuantScheme(quantization)
|
||||
base = _ONLINE_SHORTHANDS.get(quantization) if quantization else None
|
||||
|
||||
if quantization_config is None:
|
||||
quantization_config = {
|
||||
"global_scheme": scheme.value,
|
||||
}
|
||||
elif isinstance(quantization_config, OnlineQuantizationConfigArgs):
|
||||
if quantization_config.global_scheme is None:
|
||||
quantization_config.global_scheme = scheme
|
||||
elif quantization_config.global_scheme != scheme:
|
||||
raise ValueError(
|
||||
f"quantization={quantization!r} conflicts with "
|
||||
f"quantization_config.global_scheme="
|
||||
f"{quantization_config.global_scheme.value!r}. "
|
||||
f"These must match when both are specified."
|
||||
)
|
||||
elif isinstance(quantization_config, dict):
|
||||
existing = quantization_config.get("global_scheme")
|
||||
if existing is None:
|
||||
quantization_config["global_scheme"] = scheme.value
|
||||
else:
|
||||
# Coerce to enum for comparison
|
||||
existing_scheme = (
|
||||
OnlineQuantScheme(existing)
|
||||
if isinstance(existing, str)
|
||||
else existing
|
||||
)
|
||||
if existing_scheme != scheme:
|
||||
raise ValueError(
|
||||
f"quantization={quantization!r} conflicts "
|
||||
f"with quantization_config"
|
||||
f"['global_scheme']={existing!r}. "
|
||||
f"These must match when both are specified."
|
||||
)
|
||||
if quantization_config is None:
|
||||
return base
|
||||
|
||||
# Cast dict to OnlineQuantizationConfigArgs
|
||||
if isinstance(quantization_config, dict):
|
||||
quantization_config = OnlineQuantizationConfigArgs(**quantization_config)
|
||||
quantization_config = QuantizationConfigArgs(**quantization_config)
|
||||
|
||||
return quantization_config
|
||||
if base is None:
|
||||
return quantization_config
|
||||
|
||||
return QuantizationConfigArgs(
|
||||
linear=quantization_config.linear or base.linear,
|
||||
moe=quantization_config.moe or base.moe,
|
||||
ignore=quantization_config.ignore or base.ignore,
|
||||
)
|
||||
|
||||
@@ -118,7 +118,7 @@ from vllm.v1.sample.logits_processor import LogitsProcessor
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.quantization import OnlineQuantizationConfigArgs
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.model_executor.model_loader import LoadFormats
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
@@ -527,7 +527,12 @@ class EngineArgs:
|
||||
hf_overrides: HfOverrides = get_field(ModelConfig, "hf_overrides")
|
||||
tokenizer_revision: str | None = ModelConfig.tokenizer_revision
|
||||
quantization: QuantizationMethods | str | None = ModelConfig.quantization
|
||||
quantization_config: "dict[str, Any] | OnlineQuantizationConfigArgs | None" = None
|
||||
quantization_config: "dict[str, Any] | QuantizationConfigArgs | None" = None
|
||||
"""User-facing quantization configuration. Carries per-layer-kind
|
||||
QuantSpecs (linear, moe) and ignore patterns; see
|
||||
:class:`QuantizationConfigArgs`. Auto-populated from the matching online
|
||||
shorthand when `quantization` is one of the values in
|
||||
`ONLINE_QUANT_SHORTHAND_NAMES`."""
|
||||
allow_deprecated_quantization: bool = ModelConfig.allow_deprecated_quantization
|
||||
enforce_eager: bool = ModelConfig.enforce_eager
|
||||
disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
|
||||
@@ -719,9 +724,9 @@ class EngineArgs:
|
||||
if isinstance(self.ir_op_priority, dict):
|
||||
self.ir_op_priority = IrOpPriorityConfig(**self.ir_op_priority)
|
||||
|
||||
from vllm.config.quantization import resolve_online_quant_config
|
||||
from vllm.config.quantization import resolve_quantization_config
|
||||
|
||||
self.quantization_config = resolve_online_quant_config(
|
||||
self.quantization_config = resolve_quantization_config(
|
||||
self.quantization, self.quantization_config
|
||||
)
|
||||
|
||||
@@ -785,6 +790,9 @@ class EngineArgs:
|
||||
)
|
||||
model_group.add_argument("--max-model-len", **model_kwargs["max_model_len"])
|
||||
model_group.add_argument("--quantization", "-q", **model_kwargs["quantization"])
|
||||
model_group.add_argument(
|
||||
"--quantization-config", **model_kwargs["quantization_config"]
|
||||
)
|
||||
model_group.add_argument(
|
||||
"--allow-deprecated-quantization",
|
||||
**model_kwargs["allow_deprecated_quantization"],
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.config.model import (
|
||||
TokenizerMode,
|
||||
)
|
||||
from vllm.config.quantization import (
|
||||
OnlineQuantizationConfigArgs,
|
||||
QuantizationConfigArgs,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferInitRequest,
|
||||
@@ -248,9 +248,7 @@ class LLM:
|
||||
attention_config: dict[str, Any] | AttentionConfig | None = None,
|
||||
kv_cache_memory_bytes: int | None = None,
|
||||
compilation_config: int | dict[str, Any] | CompilationConfig | None = None,
|
||||
quantization_config: dict[str, Any]
|
||||
| OnlineQuantizationConfigArgs
|
||||
| None = None,
|
||||
quantization_config: dict[str, Any] | QuantizationConfigArgs | None = None,
|
||||
logits_processors: list[str | type[LogitsProcessor]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from enum import Enum
|
||||
from typing import TYPE_CHECKING, Union
|
||||
from typing import TYPE_CHECKING, Literal, Union
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm import envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
@@ -207,25 +209,38 @@ def backend_to_kernel_cls(
|
||||
raise ValueError(f"Unknown MXFP4 MoE backend: {backend.value}")
|
||||
|
||||
|
||||
def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend:
|
||||
"""Map user's moe_backend string to Mxfp4MoeBackend."""
|
||||
mapping: dict[str, Mxfp4MoeBackend] = {
|
||||
"deep_gemm": Mxfp4MoeBackend.DEEPGEMM_MXFP4,
|
||||
"flashinfer_trtllm": Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
"flashinfer_trtllm_afp8": Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
"flashinfer_cutlass": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
"flashinfer_cutlass_afp8": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
"triton": Mxfp4MoeBackend.TRITON,
|
||||
"triton_unfused": Mxfp4MoeBackend.TRITON_UNFUSED,
|
||||
"humming": Mxfp4MoeBackend.HUMMING,
|
||||
"marlin": Mxfp4MoeBackend.MARLIN,
|
||||
"aiter": Mxfp4MoeBackend.AITER_MXFP4_BF16,
|
||||
"aiter_mxfp4_fp8": Mxfp4MoeBackend.AITER_MXFP4_FP8,
|
||||
"xpu": Mxfp4MoeBackend.XPU,
|
||||
"emulation": Mxfp4MoeBackend.EMULATION,
|
||||
def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
|
||||
"""Map a moe_backend string to its candidate Mxfp4MoeBackends.
|
||||
|
||||
Vendor families return all activation variants; the caller picks one
|
||||
via ``activation_key`` and ``is_supported_config``.
|
||||
"""
|
||||
mapping: dict[str, list[Mxfp4MoeBackend]] = {
|
||||
"deep_gemm": [Mxfp4MoeBackend.DEEPGEMM_MXFP4],
|
||||
"flashinfer_trtllm": [
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
],
|
||||
"flashinfer_trtllm_afp8": [Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8],
|
||||
"flashinfer_cutlass": [
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
],
|
||||
"flashinfer_cutlass_afp8": [Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8],
|
||||
"triton": [Mxfp4MoeBackend.TRITON],
|
||||
"triton_unfused": [Mxfp4MoeBackend.TRITON_UNFUSED],
|
||||
"humming": [Mxfp4MoeBackend.HUMMING],
|
||||
"marlin": [Mxfp4MoeBackend.MARLIN],
|
||||
"aiter": [
|
||||
Mxfp4MoeBackend.AITER_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.AITER_MXFP4_FP8,
|
||||
],
|
||||
"aiter_mxfp4_fp8": [Mxfp4MoeBackend.AITER_MXFP4_FP8],
|
||||
"xpu": [Mxfp4MoeBackend.XPU],
|
||||
"emulation": [Mxfp4MoeBackend.EMULATION],
|
||||
}
|
||||
if backend := mapping.get(runner_backend):
|
||||
return backend
|
||||
if backends := mapping.get(runner_backend):
|
||||
return backends
|
||||
raise ValueError(
|
||||
f"moe_backend='{runner_backend}' is not supported for MXFP4 MoE. "
|
||||
f"Expected one of {list(mapping.keys())}."
|
||||
@@ -233,16 +248,16 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend:
|
||||
|
||||
|
||||
def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
|
||||
"""
|
||||
Get available backends in priority order based on platform and config.
|
||||
Only includes BF16 backends. MXFP8 backends are selected via env vars.
|
||||
"""
|
||||
"""Available backends in priority order, BF16-act variant before
|
||||
activation-quantized variant within each vendor family."""
|
||||
_AVAILABLE_BACKENDS = [
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
Mxfp4MoeBackend.AITER_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.AITER_MXFP4_FP8,
|
||||
Mxfp4MoeBackend.TRITON,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
# TRITON_UNFUSED has bug with MTP support
|
||||
# TODO re-enable after kernel is fixed
|
||||
# TRITON_UNFUSED
|
||||
@@ -288,6 +303,79 @@ def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
|
||||
return None # BF16 activation
|
||||
|
||||
|
||||
def _user_moe_activation_override() -> QuantKey | None:
|
||||
"""User's MoE activation override from quantization_config, or None."""
|
||||
args = get_current_vllm_config().model_config.quantization_config
|
||||
if not isinstance(args, QuantizationConfigArgs) or args.moe is None:
|
||||
return None
|
||||
return args.moe.activation
|
||||
|
||||
|
||||
def _resolve_activation_key(
|
||||
model_activation_key: QuantKey | None,
|
||||
) -> QuantKey | None:
|
||||
"""Combine the model-supplied activation key with the user override.
|
||||
Raises on conflict (both set and disagreeing)."""
|
||||
user_override = _user_moe_activation_override()
|
||||
if user_override is None:
|
||||
return model_activation_key
|
||||
if model_activation_key is None or model_activation_key == user_override:
|
||||
return user_override
|
||||
raise ValueError(
|
||||
f"checkpoint declares MoE activation={model_activation_key} but "
|
||||
f"quantization_config.moe.activation={user_override}; remove the "
|
||||
f"override or align it with the checkpoint."
|
||||
)
|
||||
|
||||
|
||||
def _make_log_backend(backend: Mxfp4MoeBackend) -> str:
|
||||
return f"Using '{backend.value}' Mxfp4 MoE backend."
|
||||
|
||||
|
||||
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
|
||||
base = (
|
||||
f"Mxfp4 MoE backend '{backend.value}' does not support the "
|
||||
f"deployment configuration"
|
||||
)
|
||||
return f"{base} since {reason}." if reason else f"{base}."
|
||||
|
||||
|
||||
def _return_or_raise(
|
||||
backend: Mxfp4MoeBackend,
|
||||
config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
scope: Literal["process", "global", "local"] = "local",
|
||||
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
reason: str | None = None
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend), scope=scope)
|
||||
return backend, k_cls
|
||||
raise ValueError(_make_log_unsupported(backend, reason))
|
||||
|
||||
|
||||
def _filter_by_activation(
|
||||
backends: list[Mxfp4MoeBackend],
|
||||
requested_activation_key: QuantKey | None,
|
||||
) -> list[Mxfp4MoeBackend]:
|
||||
"""Pick variants matching ``requested_activation_key``; without one,
|
||||
prefer BF16 if the list has any, else keep the list as-is so explicit
|
||||
non-BF16 picks (e.g. the ``_afp8`` aliases) still land."""
|
||||
if requested_activation_key is not None:
|
||||
return [
|
||||
b
|
||||
for b in backends
|
||||
if _backend_activation_key(b) == requested_activation_key
|
||||
]
|
||||
bf16 = [b for b in backends if _backend_activation_key(b) is None]
|
||||
return bf16 if bf16 else backends
|
||||
|
||||
|
||||
def select_mxfp4_moe_backend(
|
||||
config: FusedMoEConfig,
|
||||
activation_key: QuantKey | None = None,
|
||||
@@ -303,8 +391,7 @@ def select_mxfp4_moe_backend(
|
||||
|
||||
Note: Shape-specific fallbacks may still occur at runtime.
|
||||
"""
|
||||
# If activation_key is explicitly provided (e.g., W4A8), use it
|
||||
requested_activation_key = activation_key
|
||||
requested_activation_key = _resolve_activation_key(activation_key)
|
||||
|
||||
activation_format = (
|
||||
mk.FusedMoEActivationFormat.BatchedExperts
|
||||
@@ -312,67 +399,55 @@ def select_mxfp4_moe_backend(
|
||||
else mk.FusedMoEActivationFormat.Standard
|
||||
)
|
||||
|
||||
def _make_log_backend(backend: Mxfp4MoeBackend):
|
||||
return f"Using '{backend.value}' Mxfp4 MoE backend."
|
||||
|
||||
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
|
||||
if reason:
|
||||
return (
|
||||
f"Mxfp4 MoE backend '{backend.value}' does not support the "
|
||||
f"deployment configuration since {reason}."
|
||||
)
|
||||
return (
|
||||
f"Mxfp4 MoE backend '{backend.value}' does not support the "
|
||||
"deployment configuration."
|
||||
)
|
||||
|
||||
def _return_or_raise(
|
||||
backend: Mxfp4MoeBackend,
|
||||
config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
reason: str | None = None
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend))
|
||||
return backend, k_cls
|
||||
raise ValueError(_make_log_unsupported(backend, reason))
|
||||
|
||||
runner_backend = config.moe_backend
|
||||
if runner_backend != "auto":
|
||||
requested_backend = map_mxfp4_backend(runner_backend)
|
||||
if (
|
||||
activation_format == mk.FusedMoEActivationFormat.BatchedExperts
|
||||
and requested_backend == Mxfp4MoeBackend.MARLIN
|
||||
):
|
||||
requested_backend = Mxfp4MoeBackend.BATCHED_MARLIN
|
||||
# Use requested_activation_key if provided, otherwise use backend default
|
||||
act_key = (
|
||||
requested_activation_key
|
||||
if requested_activation_key is not None
|
||||
else _backend_activation_key(requested_backend)
|
||||
)
|
||||
return _return_or_raise(
|
||||
requested_backend,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
act_key,
|
||||
activation_format,
|
||||
)
|
||||
requested_backends = map_mxfp4_backend(runner_backend)
|
||||
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
requested_backends = [
|
||||
Mxfp4MoeBackend.BATCHED_MARLIN if b == Mxfp4MoeBackend.MARLIN else b
|
||||
for b in requested_backends
|
||||
]
|
||||
candidates = _filter_by_activation(requested_backends, requested_activation_key)
|
||||
if not candidates:
|
||||
raise ValueError(
|
||||
f"moe_backend={runner_backend!r} does not support "
|
||||
f"activation={requested_activation_key}; supported variants: "
|
||||
f"{[b.name for b in requested_backends]}"
|
||||
)
|
||||
last_error: Exception | None = None
|
||||
for requested_backend in candidates:
|
||||
act_key = (
|
||||
requested_activation_key
|
||||
if requested_activation_key is not None
|
||||
else _backend_activation_key(requested_backend)
|
||||
)
|
||||
try:
|
||||
return _return_or_raise(
|
||||
requested_backend,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
act_key,
|
||||
activation_format,
|
||||
)
|
||||
except ValueError as e:
|
||||
last_error = e
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
# Select kernels in order of backend.
|
||||
AVAILABLE_BACKENDS = _get_priority_backends_for_gpt_oss()
|
||||
AVAILABLE_BACKENDS = _filter_by_activation(
|
||||
_get_priority_backends_for_gpt_oss(), requested_activation_key
|
||||
)
|
||||
|
||||
# Handle explicit FlashInfer MXFP4 BF16 configuration.
|
||||
if envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16"):
|
||||
if not envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16:
|
||||
AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16)
|
||||
AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16)
|
||||
for _b in (
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
):
|
||||
if _b in AVAILABLE_BACKENDS:
|
||||
AVAILABLE_BACKENDS.remove(_b)
|
||||
else:
|
||||
if current_platform.is_device_capability(90):
|
||||
return _return_or_raise(
|
||||
@@ -481,54 +556,30 @@ def select_deepseek_v4_mxfp4_moe_backend(
|
||||
else mk.FusedMoEActivationFormat.Standard
|
||||
)
|
||||
|
||||
def _make_log_backend(backend: Mxfp4MoeBackend):
|
||||
return f"Using '{backend.value}' Mxfp4 MoE backend."
|
||||
|
||||
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
|
||||
if reason:
|
||||
return (
|
||||
f"Mxfp4 MoE backend '{backend.value}' does not support the "
|
||||
f"deployment configuration since {reason}."
|
||||
)
|
||||
return (
|
||||
f"Mxfp4 MoE backend '{backend.value}' does not support the "
|
||||
"deployment configuration."
|
||||
)
|
||||
|
||||
def _return_or_raise(
|
||||
backend: Mxfp4MoeBackend,
|
||||
config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
reason: str | None = None
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend), scope="local")
|
||||
return backend, k_cls
|
||||
raise ValueError(_make_log_unsupported(backend, reason))
|
||||
|
||||
# Honor explicit moe_backend (e.g. "marlin", "triton_unfused") before
|
||||
# falling back to the auto priority list.
|
||||
runner_backend = config.moe_backend
|
||||
if runner_backend != "auto":
|
||||
requested_backend = map_mxfp4_backend(runner_backend)
|
||||
if (
|
||||
activation_format == mk.FusedMoEActivationFormat.BatchedExperts
|
||||
and requested_backend == Mxfp4MoeBackend.MARLIN
|
||||
):
|
||||
requested_backend = Mxfp4MoeBackend.BATCHED_MARLIN
|
||||
return _return_or_raise(
|
||||
requested_backend,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
_backend_activation_key(requested_backend),
|
||||
activation_format,
|
||||
)
|
||||
requested_backends = map_mxfp4_backend(runner_backend)
|
||||
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
requested_backends = [
|
||||
Mxfp4MoeBackend.BATCHED_MARLIN if b == Mxfp4MoeBackend.MARLIN else b
|
||||
for b in requested_backends
|
||||
]
|
||||
last_error: Exception | None = None
|
||||
for requested_backend in requested_backends:
|
||||
try:
|
||||
return _return_or_raise(
|
||||
requested_backend,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
_backend_activation_key(requested_backend),
|
||||
activation_format,
|
||||
)
|
||||
except ValueError as e:
|
||||
last_error = e
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
# DeepSeek-V4 on ROCm is more accurate with the unfused Triton MXFP4 path
|
||||
# than the default AITER path. Prefer Triton-unfused for this routing mode,
|
||||
|
||||
@@ -35,10 +35,9 @@ QuantizationMethods = Literal[
|
||||
"deepseek_v4_fp8",
|
||||
"cpu_awq",
|
||||
"online",
|
||||
# Below are values of the OnlineQuantScheme enum, specified as strings to
|
||||
# avoid circular import issues. This is here to provide a shortcut where
|
||||
# the user can specify "LLM(..., quantization='fp8_per_tensor')" as
|
||||
# shorthand for creating a more complicated online quant config object
|
||||
# Below are online quant shorthand names (see vllm.config.quantization).
|
||||
# Listed here as strings to avoid a circular import; kept in sync with
|
||||
# _ONLINE_SHORTHANDS by the assertion in get_quantization_config().
|
||||
"fp8_per_tensor",
|
||||
"fp8_per_block",
|
||||
"int8_per_channel_weight_only",
|
||||
@@ -111,7 +110,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
raise ValueError(f"Invalid quantization method: {quantization}")
|
||||
|
||||
# lazy import to avoid triggering `torch.compile` too early
|
||||
from vllm.config.quantization import OnlineQuantScheme
|
||||
from vllm.config.quantization import _ONLINE_SHORTHANDS
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.models.deepseek_v4 import DeepseekV4FP8Config
|
||||
|
||||
@@ -171,16 +170,15 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
"online": OnlineQuantizationConfig,
|
||||
}
|
||||
|
||||
# Below are values of the OnlineQuantScheme enum. This is here to provide
|
||||
# a shortcut where the user can specify
|
||||
# "LLM(..., quantization='fp8_per_tensor')" as shorthand for creating a
|
||||
# more complicated online quant config object
|
||||
for scheme in OnlineQuantScheme:
|
||||
assert scheme.value not in method_to_config, (
|
||||
f"Online quant scheme {scheme.value!r} conflicts with an "
|
||||
# Register online shorthands as quantization methods so the user can
|
||||
# specify "LLM(..., quantization='fp8_per_tensor')" as shorthand for
|
||||
# creating a more complicated online quant config object.
|
||||
for shorthand in _ONLINE_SHORTHANDS:
|
||||
assert shorthand not in method_to_config, (
|
||||
f"Online quant shorthand {shorthand!r} conflicts with an "
|
||||
f"existing quantization method"
|
||||
)
|
||||
method_to_config[scheme.value] = OnlineQuantizationConfig
|
||||
method_to_config[shorthand] = OnlineQuantizationConfig
|
||||
|
||||
# Update the `method_to_config` with customized quantization methods.
|
||||
method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG)
|
||||
|
||||
@@ -5,10 +5,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.quantization import (
|
||||
OnlineQuantizationConfigArgs,
|
||||
OnlineQuantScheme,
|
||||
)
|
||||
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
@@ -41,31 +38,50 @@ from vllm.model_executor.layers.quantization.online.mxfp8 import (
|
||||
Mxfp8OnlineLinearMethod,
|
||||
Mxfp8OnlineMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp8Dynamic,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Online dispatch tables, keyed by the QuantSpec.weight QuantKey. The
|
||||
# corresponding method class handles the activation choice via its
|
||||
# `supported_activation_quant` set.
|
||||
_ONLINE_LINEAR_METHODS: dict[QuantKey, type] = {
|
||||
kFp8StaticTensorSym: Fp8PerTensorOnlineLinearMethod,
|
||||
kFp8Static128BlockSym: Fp8PerBlockOnlineLinearMethod,
|
||||
kMxfp8Dynamic: Mxfp8OnlineLinearMethod,
|
||||
}
|
||||
|
||||
_ONLINE_MOE_METHODS: dict[QuantKey, type] = {
|
||||
kFp8StaticTensorSym: Fp8PerTensorOnlineMoEMethod,
|
||||
kFp8Static128BlockSym: Fp8PerBlockOnlineMoEMethod,
|
||||
kMxfp8Dynamic: Mxfp8OnlineMoEMethod,
|
||||
kInt8StaticChannelSym: Int8OnlineMoEMethod,
|
||||
}
|
||||
|
||||
|
||||
class OnlineQuantizationConfig(QuantizationConfig):
|
||||
"""Model-level config class for online quantization (quantize fp16/bf16 weights
|
||||
"""Model-level config for online quantization (quantize fp16/bf16 weights
|
||||
during model loading, without requiring a pre-quantized checkpoint)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
args: OnlineQuantizationConfigArgs,
|
||||
args: QuantizationConfigArgs,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if (
|
||||
args.global_scheme is None
|
||||
and args.linear_scheme_override is None
|
||||
and args.moe_scheme_override is None
|
||||
):
|
||||
if args.linear is None and args.moe is None:
|
||||
raise ValueError(
|
||||
"OnlineQuantizationConfig requires at least one of "
|
||||
"global_scheme, linear_scheme_override, or "
|
||||
"moe_scheme_override to be set."
|
||||
"quantization_config.linear or quantization_config.moe "
|
||||
"to be set."
|
||||
)
|
||||
self.args = args
|
||||
self.quant_scheme = args.global_scheme
|
||||
self.ignored_layers: list[str] = args.ignore
|
||||
|
||||
@classmethod
|
||||
@@ -94,6 +110,33 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
"quantization='fp8_per_tensor'/'fp8_per_block' instead."
|
||||
)
|
||||
|
||||
def _dispatch(
|
||||
self,
|
||||
spec: QuantSpec | None,
|
||||
table: dict[QuantKey, type],
|
||||
layer: torch.nn.Module,
|
||||
) -> "QuantizeMethodBase | None":
|
||||
if spec is None or spec.weight is None:
|
||||
return None
|
||||
cls = table.get(spec.weight)
|
||||
if cls is None:
|
||||
raise ValueError(
|
||||
f"online quantization for {type(layer).__name__} with "
|
||||
f"weight={spec.weight} is not supported; supported weight "
|
||||
f"keys: {sorted(str(k) for k in table)}"
|
||||
)
|
||||
# Online method classes pick their own activation format internally.
|
||||
# Per-class activation overrides are not yet wired through; reject
|
||||
# explicit overrides until the relevant method class opts in.
|
||||
if spec.activation is not None:
|
||||
raise ValueError(
|
||||
f"activation override (activation={spec.activation}) is not "
|
||||
f"yet supported for online {cls.__name__}"
|
||||
)
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return cls(layer=layer)
|
||||
return cls()
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
@@ -104,20 +147,8 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
|
||||
linear_scheme = self.args.linear_scheme_override or self.args.global_scheme
|
||||
if linear_scheme == OnlineQuantScheme.INT8_PER_CHANNEL_WEIGHT_ONLY:
|
||||
logger.warning_once(
|
||||
"INT8 online quantization only quantizes MoE expert "
|
||||
"weights. linear layers remain in full precision."
|
||||
)
|
||||
return UnquantizedLinearMethod()
|
||||
elif linear_scheme == OnlineQuantScheme.FP8_PER_BLOCK:
|
||||
return Fp8PerBlockOnlineLinearMethod()
|
||||
elif linear_scheme == OnlineQuantScheme.MXFP8:
|
||||
return Mxfp8OnlineLinearMethod()
|
||||
else:
|
||||
return Fp8PerTensorOnlineLinearMethod()
|
||||
method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer)
|
||||
return method if method is not None else UnquantizedLinearMethod()
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
if should_ignore_layer(
|
||||
prefix,
|
||||
@@ -125,14 +156,10 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
|
||||
moe_scheme = self.args.moe_scheme_override or self.args.global_scheme
|
||||
if moe_scheme == OnlineQuantScheme.INT8_PER_CHANNEL_WEIGHT_ONLY:
|
||||
return Int8OnlineMoEMethod(layer=layer)
|
||||
elif moe_scheme == OnlineQuantScheme.FP8_PER_BLOCK:
|
||||
return Fp8PerBlockOnlineMoEMethod(layer=layer)
|
||||
elif moe_scheme == OnlineQuantScheme.MXFP8:
|
||||
return Mxfp8OnlineMoEMethod(layer=layer)
|
||||
else:
|
||||
return Fp8PerTensorOnlineMoEMethod(layer=layer)
|
||||
method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer)
|
||||
return (
|
||||
method
|
||||
if method is not None
|
||||
else UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -300,12 +300,10 @@ def get_quant_config(
|
||||
)
|
||||
|
||||
if hf_quant_config is not None:
|
||||
if model_config.quantization_config is not None:
|
||||
raise ValueError(
|
||||
"Setting `quantization_config` for online "
|
||||
"quantization when the model checkpoint already "
|
||||
"has a `quantization_config` is not supported"
|
||||
)
|
||||
# `model_config.quantization_config` may be set alongside a checkpoint
|
||||
# quant config: the checkpoint determines `quant_cls`, and the user's
|
||||
# QuantizationConfigArgs is consulted by individual quant methods
|
||||
# (e.g. for activation overrides via the MXFP4 oracle).
|
||||
|
||||
# For modelopt_mixed, config.json's quantization_config may or may
|
||||
# not contain the per-layer quantized_layers map. Newer checkpoints
|
||||
@@ -330,12 +328,6 @@ def get_quant_config(
|
||||
quantization_config_file = hf_overrides.get("quantization_config_file", None)
|
||||
if quantization_config_file is not None:
|
||||
if hasattr(quant_cls, "from_config_file"):
|
||||
if model_config.quantization_config is not None:
|
||||
raise ValueError(
|
||||
"Setting `quantization_config` for online "
|
||||
"quantization when the model checkpoint already "
|
||||
"has a `quantization_config` is not supported"
|
||||
)
|
||||
return quant_cls.from_config_file(quantization_config_file)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
@@ -346,12 +338,6 @@ def get_quant_config(
|
||||
quantization_config_json = hf_overrides.get("quantization_config_dict_json", None)
|
||||
if quantization_config_json is not None:
|
||||
if hasattr(quant_cls, "from_config_dict_json"):
|
||||
if model_config.quantization_config is not None:
|
||||
raise ValueError(
|
||||
"Setting `quantization_config` for online "
|
||||
"quantization when the model checkpoint already "
|
||||
"has a `quantization_config` is not supported"
|
||||
)
|
||||
return quant_cls.from_config_dict_json(quantization_config_json)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
@@ -360,17 +346,15 @@ def get_quant_config(
|
||||
f"{quant_cls}"
|
||||
)
|
||||
|
||||
# Online quantization doesn't read from checkpoint configs — it quantizes
|
||||
# Online quantization doesn't read from checkpoint configs - it quantizes
|
||||
# fp16/bf16 weights on the fly during loading.
|
||||
if model_config.quantization_config is not None:
|
||||
from vllm.config.quantization import OnlineQuantizationConfigArgs
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
|
||||
assert isinstance(
|
||||
model_config.quantization_config, OnlineQuantizationConfigArgs
|
||||
)
|
||||
assert isinstance(model_config.quantization_config, QuantizationConfigArgs)
|
||||
return OnlineQuantizationConfig(args=model_config.quantization_config)
|
||||
|
||||
# Inflight BNB quantization
|
||||
|
||||
Reference in New Issue
Block a user