diff --git a/docs/features/quantization/online.md b/docs/features/quantization/online.md index 913da250ee..4b9571b38b 100644 --- a/docs/features/quantization/online.md +++ b/docs/features/quantization/online.md @@ -42,18 +42,59 @@ vllm serve meta-llama/Llama-3.1-8B --quantization mxfp8 For fine-grained control, use a `quantization_config` dictionary. +### Schema + +```yaml +quantization_config: + linear: + weight: # see QUANT_KEY_NAMES in vllm/config/quantization.py + activation: + moe: + weight: + activation: + ignore: [, ...] +``` + +`linear` and `moe` accept a full `{weight, activation}` dict, or a bare +string. A string resolves first against the `--quantization` shorthands +(taking the matching layer-kind slot), then against `QUANT_KEY_NAMES` as a +weight name. Unset fields fall back to the `--quantization` shorthand's +defaults, or for already-quantized checkpoints to whatever the checkpoint +declares. + +The CLI accepts the same shape as JSON or as dotted keys: + +```bash +vllm serve --quantization-config '{"moe":{"activation":"mxfp8"}}' +vllm serve --quantization-config.moe.activation mxfp8 +``` + +### Activation overrides on already-quantized checkpoints + +For checkpoint-quantized models, `quantization_config` lets you pick an +activation format independently of the baked-in weights. The supported +overrides are checkpoint-specific; today this is wired up for MXFP4 MoE +checkpoints (gpt-oss) where you can opt into FP8 activations: + +```bash +vllm serve openai/gpt-oss-20b --quantization-config.moe.activation mxfp8 +``` + +Combine with `--moe-backend` to pin a specific kernel family. + ### Separate Schemes for Dense and MoE Layers -You can apply different quantization schemes to dense linear layers and MoE expert layers: +You can apply different quantization schemes to dense linear layers and MoE expert layers via the `linear` and `moe` fields. Each accepts either a full spec dict, or a bare string naming an online shorthand (e.g. `"fp8_per_block"`) or weight format (e.g. `"fp8_per_block_static"`); fields not set fall back to the shorthand defaults. ```python from vllm import LLM +# Linear: per-block FP8; MoE: per-tensor FP8 (inherited from the shorthand) llm = LLM( "ibm-granite/granite-3.0-1b-a400m-base", quantization="fp8_per_tensor", quantization_config={ - "linear_scheme_override": "fp8_per_block", + "linear": "fp8_per_block", }, ) ``` @@ -63,11 +104,12 @@ Or, ```python from vllm import LLM +# Linear: per-tensor FP8 (inherited); MoE: per-block FP8 llm = LLM( "ibm-granite/granite-3.0-1b-a400m-base", quantization="fp8_per_tensor", quantization_config={ - "moe_scheme_override": "fp8_per_block", + "moe": "fp8_per_block", }, ) ``` diff --git a/tests/compile/fusions_e2e/conftest.py b/tests/compile/fusions_e2e/conftest.py index b4922a3fdc..3a06087472 100644 --- a/tests/compile/fusions_e2e/conftest.py +++ b/tests/compile/fusions_e2e/conftest.py @@ -103,13 +103,6 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn): if not supports_trtllm_attention(): matches = matches._replace(attn_quant_fusion=0) - # TODO: remove this after finishing migration from envs to model kwargs - if model_name == "openai/gpt-oss-20b": - from .common import is_blackwell - - if is_blackwell(): - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "1") - # Disable, compile cache to make sure custom passes run. # Otherwise, we can't verify fusion happened through the logs. monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1") diff --git a/tests/compile/fusions_e2e/models.py b/tests/compile/fusions_e2e/models.py index 3d73373cb9..32f1ea3506 100644 --- a/tests/compile/fusions_e2e/models.py +++ b/tests/compile/fusions_e2e/models.py @@ -214,4 +214,9 @@ gpt_oss_20b = ModelFusionInfo( sequence_parallel=n_layers * 2 + 1, async_tp=n_layers * 2, ), + model_kwargs=( + {"quantization_config": {"moe": {"activation": "mxfp8"}}} + if is_blackwell() + else {} + ), ) diff --git a/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml b/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml index 23ec14819e..5a8e6534a6 100644 --- a/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml +++ b/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml @@ -3,6 +3,4 @@ model_name: "openai/gpt-oss-20b" metric_threshold: 0.568 reasoning_effort: "low" -server_args: "--tensor-parallel-size 2" -env: - VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS: "1" +server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_cutlass --quantization-config.moe.activation mxfp8" diff --git a/tests/evals/gpt_oss/configs/gpt-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml b/tests/evals/gpt_oss/configs/gpt-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml index 4cea743490..895f72d099 100644 --- a/tests/evals/gpt_oss/configs/gpt-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml +++ b/tests/evals/gpt_oss/configs/gpt-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml @@ -3,6 +3,4 @@ model_name: "openai/gpt-oss-20b" metric_threshold: 0.568 reasoning_effort: "low" -server_args: "--tensor-parallel-size 2" -env: - VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: "1" +server_args: "--tensor-parallel-size 2 --quantization-config.moe.activation mxfp8" diff --git a/tests/quantization/test_blackwell_moe.py b/tests/quantization/test_blackwell_moe.py index 056191eab5..8c525149ca 100644 --- a/tests/quantization/test_blackwell_moe.py +++ b/tests/quantization/test_blackwell_moe.py @@ -190,13 +190,27 @@ def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch): def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", "1") - can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT) + can_initialize( + "openai/gpt-oss-20b", + hf_overrides=HF_OVERRIDE_TEXT, + extra_args=[ + "--moe-backend", + "flashinfer_cutlass", + "--quantization-config.moe.activation", + "mxfp8", + ], + ) def test_gptoss_mxfp4mxfp8_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "1") - can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT) + can_initialize( + "openai/gpt-oss-20b", + hf_overrides=HF_OVERRIDE_TEXT, + extra_args=[ + "--quantization-config.moe.activation", + "mxfp8", + ], + ) def test_gptoss_eager(monkeypatch: pytest.MonkeyPatch): diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index 89f9676e43..0254da79e1 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -40,13 +40,12 @@ from vllm.platforms import current_platform Fp8PerBlockOnlineLinearMethod, Fp8PerBlockOnlineMoEMethod, ), - # quantization='online with linear_scheme_override and - # moe_scheme_override + # quantization='online' with per-layer-kind overrides ( "online", { - "linear_scheme_override": "fp8_per_block", - "moe_scheme_override": "fp8_per_tensor", + "linear": "fp8_per_block", + "moe": "fp8_per_tensor", }, Fp8PerBlockOnlineLinearMethod, Fp8PerTensorOnlineMoEMethod, diff --git a/tests/quantization/test_quantization_config_args.py b/tests/quantization/test_quantization_config_args.py new file mode 100644 index 0000000000..7e881d6b0b --- /dev/null +++ b/tests/quantization/test_quantization_config_args.py @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for QuantizationConfigArgs parsing.""" + +import pytest + +from vllm.config.quantization import ( + QUANT_KEY_NAMES, + QuantizationConfigArgs, + QuantSpec, + resolve_quantization_config, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kFp8Dynamic128Sym, + kFp8DynamicTokenSym, + kFp8Static128BlockSym, + kFp8StaticTensorSym, + kInt8StaticChannelSym, + kMxfp8Dynamic, +) + +# ---- QuantSpec ------------------------------------------------------------ + + +def test_quant_spec_resolves_string_to_quant_key(): + spec = QuantSpec(weight="mxfp8", activation="fp8_per_token") + assert spec.weight == kMxfp8Dynamic + assert spec.activation == kFp8DynamicTokenSym + + +def test_quant_spec_accepts_quant_key_directly(): + spec = QuantSpec(weight=kFp8StaticTensorSym) + assert spec.weight is kFp8StaticTensorSym + assert spec.activation is None + + +def test_quant_spec_rejects_unknown_name(): + with pytest.raises(ValueError, match="unknown quantization name"): + QuantSpec(weight="not_a_real_format") + + +# ---- QuantizationConfigArgs string shorthand on linear/moe ---------------- + + +def test_args_linear_string_resolves_via_quant_key_names(): + # A bare QUANT_KEY_NAMES entry desugars to QuantSpec(weight=). + args = QuantizationConfigArgs(linear="fp8_per_block_static") + assert args.linear == QuantSpec(weight=kFp8Static128BlockSym) + assert args.moe is None + + +def test_args_moe_string_resolves_via_online_shorthand(): + # An online-shorthand name pulls the matching slot from _ONLINE_SHORTHANDS + # (so `linear: "fp8_per_block"` and `moe: "fp8_per_block"` produce the + # same per-layer-kind spec the `--quantization fp8_per_block` shorthand + # would). + args = QuantizationConfigArgs(moe="fp8_per_block") + assert args.moe == QuantSpec(weight=kFp8Static128BlockSym) + + +def test_args_string_shorthand_missing_slot_raises(): + # int8_per_channel_weight_only sets only `moe`; using it on `linear` + # has no defined spec and should raise rather than silently no-op. + with pytest.raises(ValueError, match="does not define a linear spec"): + QuantizationConfigArgs(linear="int8_per_channel_weight_only") + + +def test_args_accepts_dict_form(): + args = QuantizationConfigArgs(moe={"activation": "mxfp8"}) + assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic) + + +# ---- resolve_quantization_config ----------------------------------------- + + +def test_resolve_shorthand_only_populates_both_slots(): + args = resolve_quantization_config("fp8_per_block", None) + assert args.linear == QuantSpec(weight=kFp8Static128BlockSym) + assert args.moe == QuantSpec(weight=kFp8Static128BlockSym) + + +def test_resolve_int8_shorthand_leaves_linear_unset(): + # int8_per_channel_weight_only is MoE-only; linear stays None so that + # OnlineQuantizationConfig leaves Linear layers in full precision. + args = resolve_quantization_config("int8_per_channel_weight_only", None) + assert args.linear is None + assert args.moe == QuantSpec(weight=kInt8StaticChannelSym) + + +def test_resolve_quantization_config_only(): + # When only `quantization_config` is given (e.g. for an already-quantized + # checkpoint that needs an activation override), it's returned as-is. + args = resolve_quantization_config(None, {"moe": {"activation": "mxfp8"}}) + assert args.linear is None + assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic) + + +def test_resolve_merges_explicit_over_shorthand(): + # Explicit linear in quantization_config wins; moe falls back to the + # shorthand's slot. + args = resolve_quantization_config( + "fp8_per_tensor", + {"linear": "fp8_per_block"}, + ) + assert args.linear == QuantSpec(weight=kFp8Static128BlockSym) + assert args.moe == QuantSpec(weight=kFp8StaticTensorSym) + + +def test_resolve_rejects_quantization_config_with_non_shorthand_quant(): + # If --quantization names something other than an online shorthand, + # quantization_config is not allowed via this path (checkpoint quant + # paths read it directly off ModelConfig instead). + with pytest.raises(ValueError, match="quantization_config is only supported"): + resolve_quantization_config("gptq", {"linear": "fp8_per_block"}) + + +# ---- QUANT_KEY_NAMES coverage -------------------------------------------- + + +def test_quant_key_names_round_trip(): + # Every advertised name should round-trip through QuantSpec without error + # and produce the same QuantKey it maps to. + for name, expected in QUANT_KEY_NAMES.items(): + assert QuantSpec(weight=name).weight == expected, name + assert QuantSpec(activation=name).activation == expected, name + + +def test_static_block_weight_paired_with_dynamic_block_activation(): + # The block-FP8 shorthand pair: 128x128 static weights + 1x128 dynamic + # activations. Pinning this so renames in QUANT_KEY_NAMES don't quietly + # rewire the kernel dispatch. + spec = QuantSpec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic") + assert spec.weight == kFp8Static128BlockSym + assert spec.activation == kFp8Dynamic128Sym diff --git a/vllm/config/model.py b/vllm/config/model.py index 193f48a545..1aeb21e4e7 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -21,7 +21,7 @@ from vllm.config.multimodal import ( MultiModalConfig, ) from vllm.config.pooler import PoolerConfig -from vllm.config.quantization import OnlineQuantizationConfigArgs +from vllm.config.quantization import QuantizationConfigArgs from vllm.config.scheduler import RunnerType from vllm.config.utils import config, getattr_iter from vllm.logger import init_logger @@ -206,10 +206,11 @@ class ModelConfig: `quantization_config` attribute in the model config file. If that is `None`, we assume the model weights are not quantized and use `dtype` to determine the data type of the weights.""" - quantization_config: dict[str, Any] | OnlineQuantizationConfigArgs | None = None - """Arguments for online quantization. - Auto-created when `quantization` equals to one of the string values of - the `OnlineQuantScheme` enum.""" + quantization_config: dict[str, Any] | QuantizationConfigArgs | None = None + """User-facing quantization configuration. Carries per-layer-kind specs + (linear, moe) and ignore patterns; see :class:`QuantizationConfigArgs`. + Auto-populated from the matching online shorthand when `quantization` is + one of the values in `ONLINE_QUANT_SHORTHAND_NAMES`.""" allow_deprecated_quantization: bool = False """Whether to allow deprecated quantization methods.""" enforce_eager: bool = False diff --git a/vllm/config/quantization.py b/vllm/config/quantization.py index 05e91f9402..b726d4ac23 100644 --- a/vllm/config/quantization.py +++ b/vllm/config/quantization.py @@ -1,126 +1,175 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from enum import Enum -from typing import Any +from typing import Annotated, Any -from pydantic import Field, field_validator +from pydantic import Field, GetPydanticSchema, ValidationInfo, field_validator +from pydantic_core import core_schema from vllm.config.utils import config +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kFp8Dynamic128Sym, + kFp8DynamicTensorSym, + kFp8DynamicTokenSym, + kFp8Static128BlockSym, + kFp8StaticTensorSym, + kInt8StaticChannelSym, + kMxfp4Dynamic, + kMxfp8Dynamic, +) + +# User-facing names addressable from quantization_config. +QUANT_KEY_NAMES: dict[str, QuantKey] = { + "fp8_per_tensor_static": kFp8StaticTensorSym, + "fp8_per_tensor_dynamic": kFp8DynamicTensorSym, + "fp8_per_token": kFp8DynamicTokenSym, + "fp8_per_block_static": kFp8Static128BlockSym, + "fp8_per_block_dynamic": kFp8Dynamic128Sym, + "mxfp8": kMxfp8Dynamic, + "mxfp4": kMxfp4Dynamic, + "int8_per_channel_static": kInt8StaticChannelSym, +} -class OnlineQuantScheme(Enum): - """Supported online quantization schemes.""" +def _coerce_quant_key(v: Any) -> QuantKey | None: + if v is None or isinstance(v, QuantKey): + return v + if not isinstance(v, str): + raise TypeError(f"expected str or QuantKey, got {type(v).__name__}") + try: + return QUANT_KEY_NAMES[v] + except KeyError: + raise ValueError( + f"unknown quantization name {v!r}; " + f"expected one of {sorted(QUANT_KEY_NAMES)}" + ) from None - # fp8, weights and activations scaled per-tensor - FP8_PER_TENSOR = "fp8_per_tensor" - # fp8, activations scaled in blocks of 1x128 elements, weights scaled in - # blocks of 128x128 elements (popularized by DeepSeek) - FP8_PER_BLOCK = "fp8_per_block" - - # int8, weight-only per-channel quantization for MoE expert weights. - # Linear layers remain unquantized. - INT8_PER_CHANNEL_WEIGHT_ONLY = "int8_per_channel_weight_only" - - # mxfp8, weights scaled in blocks of 1x32 elements (microscaling FP8) - MXFP8 = "mxfp8" +# Stop pydantic from introspecting QuantKey: it transitively contains a +# NamedTuple with `ClassVar[GroupShape]` declarations that pydantic refuses. +QuantKeyField = Annotated[ + QuantKey | None, + GetPydanticSchema( + lambda _src, _handler: core_schema.no_info_plain_validator_function( + _coerce_quant_key + ) + ), +] @config -class OnlineQuantizationConfigArgs: - """Configuration for online quantization. +class QuantSpec: + """Quantization spec for one layer kind (linear or MoE). - Controls how ``OnlineQuantizationConfig`` is applied to a model. - At least one of ``global_scheme``, ``linear_scheme_override``, or - ``moe_scheme_override`` must be set. + `None` on either side means the method class falls back to its own default + (typically inherited from the checkpoint, or unquantized for online). """ - global_scheme: OnlineQuantScheme | None = None - """Quantization scheme applied to every supported layer.""" + weight: QuantKeyField = None + """Weight quantization key, or a name from QUANT_KEY_NAMES.""" - linear_scheme_override: OnlineQuantScheme | None = None - """Quantization scheme override for ``LinearBase`` layers.""" + activation: QuantKeyField = None + """Activation quantization key, or a name from QUANT_KEY_NAMES.""" - moe_scheme_override: OnlineQuantScheme | None = None - """Quantization scheme override for ``FusedMoE`` layers.""" + +@config +class QuantizationConfigArgs: + """User-facing quantization configuration. + + See `docs/features/quantization/online.md` for the schema and shorthand + string forms accepted on `linear` and `moe`. + """ + + linear: QuantSpec | None = None + """Spec applied to ``LinearBase`` layers.""" + + moe: QuantSpec | None = None + """Spec applied to ``FusedMoE`` layers.""" ignore: list[str] = Field(default_factory=list) - """Layers to skip quantization for. Supports exact names and regex - patterns with ``re:`` prefix (e.g. ``re:.*attn.*``), consistent with - compressed_tensors layer skipping.""" + """Layers to skip quantization for.""" - @field_validator( - "global_scheme", "linear_scheme_override", "moe_scheme_override", mode="before" - ) + @field_validator("linear", "moe", mode="before") @classmethod - def _coerce_scheme( - cls, v: str | OnlineQuantScheme | None - ) -> OnlineQuantScheme | None: - if isinstance(v, str): - return OnlineQuantScheme(v) - return v + def _coerce_spec(cls, v: Any, info: ValidationInfo) -> Any: + if not isinstance(v, str): + return v + field_name = info.field_name + assert field_name is not None + if v in _ONLINE_SHORTHANDS: + spec = getattr(_ONLINE_SHORTHANDS[v], field_name) + if spec is None: + raise ValueError( + f"online shorthand {v!r} does not define a {field_name} spec" + ) + return spec + return QuantSpec(weight=_coerce_quant_key(v)) -def resolve_online_quant_config( +# CLI shorthands accepted by `--quantization`. Each desugars to a full +# QuantizationConfigArgs; activation overrides go through quantization_config. +_ONLINE_SHORTHANDS: dict[str, QuantizationConfigArgs] = { + "fp8_per_tensor": QuantizationConfigArgs( + linear=QuantSpec(weight=kFp8StaticTensorSym), + moe=QuantSpec(weight=kFp8StaticTensorSym), + ), + "fp8_per_block": QuantizationConfigArgs( + linear=QuantSpec(weight=kFp8Static128BlockSym), + moe=QuantSpec(weight=kFp8Static128BlockSym), + ), + "mxfp8": QuantizationConfigArgs( + linear=QuantSpec(weight=kMxfp8Dynamic), + moe=QuantSpec(weight=kMxfp8Dynamic), + ), + # INT8 weight-only on MoE; linear stays unquantized (no `linear` field). + "int8_per_channel_weight_only": QuantizationConfigArgs( + moe=QuantSpec(weight=kInt8StaticChannelSym), + ), +} + + +# Names accepted by `--quantization`; "online" means "use quantization_config". +ONLINE_QUANT_SHORTHAND_NAMES: tuple[str, ...] = ( + *_ONLINE_SHORTHANDS.keys(), + "online", +) + + +def resolve_quantization_config( quantization: str | None, - quantization_config: dict[str, Any] | OnlineQuantizationConfigArgs | None, -) -> OnlineQuantizationConfigArgs | None: - """Resolve online quant scheme shorthand into a quantization config. + quantization_config: dict[str, Any] | QuantizationConfigArgs | None, +) -> QuantizationConfigArgs | None: + """Resolve `--quantization` shorthand and `--quantization-config` into a + QuantizationConfigArgs. - If ``quantization`` is an online quant scheme (e.g. ``'fp8_per_tensor'``), - ensures ``quantization_config`` has a matching ``global_scheme`` and casts - it to :class:`OnlineQuantizationConfigArgs` if needed. + `quantization` is a CLI shorthand that desugars into a base config via + `_ONLINE_SHORTHANDS`. `quantization_config` is a dict or pre-built args + object. When both are given, fields explicitly set in `quantization_config` + take precedence over the shorthand. """ - online_quant_values = {s.value for s in OnlineQuantScheme} - valid_quantization_values = online_quant_values | {"online"} - if quantization not in valid_quantization_values: + if quantization is not None and quantization not in ONLINE_QUANT_SHORTHAND_NAMES: if quantization_config is not None: raise ValueError( - f"quantization_config is only supported when quantization " - f"is one of {sorted(valid_quantization_values)}, " + f"quantization_config is only supported when quantization is " + f"one of {sorted(ONLINE_QUANT_SHORTHAND_NAMES)}, " f"got quantization={quantization!r}" ) return None - if quantization in online_quant_values: - scheme = OnlineQuantScheme(quantization) + base = _ONLINE_SHORTHANDS.get(quantization) if quantization else None - if quantization_config is None: - quantization_config = { - "global_scheme": scheme.value, - } - elif isinstance(quantization_config, OnlineQuantizationConfigArgs): - if quantization_config.global_scheme is None: - quantization_config.global_scheme = scheme - elif quantization_config.global_scheme != scheme: - raise ValueError( - f"quantization={quantization!r} conflicts with " - f"quantization_config.global_scheme=" - f"{quantization_config.global_scheme.value!r}. " - f"These must match when both are specified." - ) - elif isinstance(quantization_config, dict): - existing = quantization_config.get("global_scheme") - if existing is None: - quantization_config["global_scheme"] = scheme.value - else: - # Coerce to enum for comparison - existing_scheme = ( - OnlineQuantScheme(existing) - if isinstance(existing, str) - else existing - ) - if existing_scheme != scheme: - raise ValueError( - f"quantization={quantization!r} conflicts " - f"with quantization_config" - f"['global_scheme']={existing!r}. " - f"These must match when both are specified." - ) + if quantization_config is None: + return base - # Cast dict to OnlineQuantizationConfigArgs if isinstance(quantization_config, dict): - quantization_config = OnlineQuantizationConfigArgs(**quantization_config) + quantization_config = QuantizationConfigArgs(**quantization_config) - return quantization_config + if base is None: + return quantization_config + + return QuantizationConfigArgs( + linear=quantization_config.linear or base.linear, + moe=quantization_config.moe or base.moe, + ignore=quantization_config.ignore or base.ignore, + ) diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index c685ea674f..ad492fde4e 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -118,7 +118,7 @@ from vllm.v1.sample.logits_processor import LogitsProcessor from vllm.version import __version__ as VLLM_VERSION if TYPE_CHECKING: - from vllm.config.quantization import OnlineQuantizationConfigArgs + from vllm.config.quantization import QuantizationConfigArgs from vllm.model_executor.layers.quantization import QuantizationMethods from vllm.model_executor.model_loader import LoadFormats from vllm.usage.usage_lib import UsageContext @@ -527,7 +527,12 @@ class EngineArgs: hf_overrides: HfOverrides = get_field(ModelConfig, "hf_overrides") tokenizer_revision: str | None = ModelConfig.tokenizer_revision quantization: QuantizationMethods | str | None = ModelConfig.quantization - quantization_config: "dict[str, Any] | OnlineQuantizationConfigArgs | None" = None + quantization_config: "dict[str, Any] | QuantizationConfigArgs | None" = None + """User-facing quantization configuration. Carries per-layer-kind + QuantSpecs (linear, moe) and ignore patterns; see + :class:`QuantizationConfigArgs`. Auto-populated from the matching online + shorthand when `quantization` is one of the values in + `ONLINE_QUANT_SHORTHAND_NAMES`.""" allow_deprecated_quantization: bool = ModelConfig.allow_deprecated_quantization enforce_eager: bool = ModelConfig.enforce_eager disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce @@ -719,9 +724,9 @@ class EngineArgs: if isinstance(self.ir_op_priority, dict): self.ir_op_priority = IrOpPriorityConfig(**self.ir_op_priority) - from vllm.config.quantization import resolve_online_quant_config + from vllm.config.quantization import resolve_quantization_config - self.quantization_config = resolve_online_quant_config( + self.quantization_config = resolve_quantization_config( self.quantization, self.quantization_config ) @@ -785,6 +790,9 @@ class EngineArgs: ) model_group.add_argument("--max-model-len", **model_kwargs["max_model_len"]) model_group.add_argument("--quantization", "-q", **model_kwargs["quantization"]) + model_group.add_argument( + "--quantization-config", **model_kwargs["quantization_config"] + ) model_group.add_argument( "--allow-deprecated-quantization", **model_kwargs["allow_deprecated_quantization"], diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 4e8d45301b..e7d4cd88d5 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -35,7 +35,7 @@ from vllm.config.model import ( TokenizerMode, ) from vllm.config.quantization import ( - OnlineQuantizationConfigArgs, + QuantizationConfigArgs, ) from vllm.distributed.weight_transfer.base import ( WeightTransferInitRequest, @@ -248,9 +248,7 @@ class LLM: attention_config: dict[str, Any] | AttentionConfig | None = None, kv_cache_memory_bytes: int | None = None, compilation_config: int | dict[str, Any] | CompilationConfig | None = None, - quantization_config: dict[str, Any] - | OnlineQuantizationConfigArgs - | None = None, + quantization_config: dict[str, Any] | QuantizationConfigArgs | None = None, logits_processors: list[str | type[LogitsProcessor]] | None = None, **kwargs: Any, ) -> None: diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 3b374c39a9..6e233f4c39 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -1,13 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from enum import Enum -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Literal, Union import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm import envs +from vllm.config import get_current_vllm_config from vllm.config.kernel import MoEBackend +from vllm.config.quantization import QuantizationConfigArgs from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( FusedMoEConfig, @@ -207,25 +209,38 @@ def backend_to_kernel_cls( raise ValueError(f"Unknown MXFP4 MoE backend: {backend.value}") -def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend: - """Map user's moe_backend string to Mxfp4MoeBackend.""" - mapping: dict[str, Mxfp4MoeBackend] = { - "deep_gemm": Mxfp4MoeBackend.DEEPGEMM_MXFP4, - "flashinfer_trtllm": Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16, - "flashinfer_trtllm_afp8": Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8, - "flashinfer_cutlass": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16, - "flashinfer_cutlass_afp8": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8, - "triton": Mxfp4MoeBackend.TRITON, - "triton_unfused": Mxfp4MoeBackend.TRITON_UNFUSED, - "humming": Mxfp4MoeBackend.HUMMING, - "marlin": Mxfp4MoeBackend.MARLIN, - "aiter": Mxfp4MoeBackend.AITER_MXFP4_BF16, - "aiter_mxfp4_fp8": Mxfp4MoeBackend.AITER_MXFP4_FP8, - "xpu": Mxfp4MoeBackend.XPU, - "emulation": Mxfp4MoeBackend.EMULATION, +def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]: + """Map a moe_backend string to its candidate Mxfp4MoeBackends. + + Vendor families return all activation variants; the caller picks one + via ``activation_key`` and ``is_supported_config``. + """ + mapping: dict[str, list[Mxfp4MoeBackend]] = { + "deep_gemm": [Mxfp4MoeBackend.DEEPGEMM_MXFP4], + "flashinfer_trtllm": [ + Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16, + Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8, + ], + "flashinfer_trtllm_afp8": [Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8], + "flashinfer_cutlass": [ + Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16, + Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8, + ], + "flashinfer_cutlass_afp8": [Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8], + "triton": [Mxfp4MoeBackend.TRITON], + "triton_unfused": [Mxfp4MoeBackend.TRITON_UNFUSED], + "humming": [Mxfp4MoeBackend.HUMMING], + "marlin": [Mxfp4MoeBackend.MARLIN], + "aiter": [ + Mxfp4MoeBackend.AITER_MXFP4_BF16, + Mxfp4MoeBackend.AITER_MXFP4_FP8, + ], + "aiter_mxfp4_fp8": [Mxfp4MoeBackend.AITER_MXFP4_FP8], + "xpu": [Mxfp4MoeBackend.XPU], + "emulation": [Mxfp4MoeBackend.EMULATION], } - if backend := mapping.get(runner_backend): - return backend + if backends := mapping.get(runner_backend): + return backends raise ValueError( f"moe_backend='{runner_backend}' is not supported for MXFP4 MoE. " f"Expected one of {list(mapping.keys())}." @@ -233,16 +248,16 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend: def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]: - """ - Get available backends in priority order based on platform and config. - Only includes BF16 backends. MXFP8 backends are selected via env vars. - """ + """Available backends in priority order, BF16-act variant before + activation-quantized variant within each vendor family.""" _AVAILABLE_BACKENDS = [ Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16, + Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8, Mxfp4MoeBackend.AITER_MXFP4_BF16, Mxfp4MoeBackend.AITER_MXFP4_FP8, Mxfp4MoeBackend.TRITON, Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16, + Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8, # TRITON_UNFUSED has bug with MTP support # TODO re-enable after kernel is fixed # TRITON_UNFUSED @@ -288,6 +303,79 @@ def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None: return None # BF16 activation +def _user_moe_activation_override() -> QuantKey | None: + """User's MoE activation override from quantization_config, or None.""" + args = get_current_vllm_config().model_config.quantization_config + if not isinstance(args, QuantizationConfigArgs) or args.moe is None: + return None + return args.moe.activation + + +def _resolve_activation_key( + model_activation_key: QuantKey | None, +) -> QuantKey | None: + """Combine the model-supplied activation key with the user override. + Raises on conflict (both set and disagreeing).""" + user_override = _user_moe_activation_override() + if user_override is None: + return model_activation_key + if model_activation_key is None or model_activation_key == user_override: + return user_override + raise ValueError( + f"checkpoint declares MoE activation={model_activation_key} but " + f"quantization_config.moe.activation={user_override}; remove the " + f"override or align it with the checkpoint." + ) + + +def _make_log_backend(backend: Mxfp4MoeBackend) -> str: + return f"Using '{backend.value}' Mxfp4 MoE backend." + + +def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str: + base = ( + f"Mxfp4 MoE backend '{backend.value}' does not support the " + f"deployment configuration" + ) + return f"{base} since {reason}." if reason else f"{base}." + + +def _return_or_raise( + backend: Mxfp4MoeBackend, + config: FusedMoEConfig, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + activation_format: mk.FusedMoEActivationFormat, + scope: Literal["process", "global", "local"] = "local", +) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]: + reason: str | None = None + for k_cls in backend_to_kernel_cls(backend): + supported, reason = k_cls.is_supported_config( + k_cls, config, weight_key, activation_key, activation_format + ) + if supported: + logger.info_once(_make_log_backend(backend), scope=scope) + return backend, k_cls + raise ValueError(_make_log_unsupported(backend, reason)) + + +def _filter_by_activation( + backends: list[Mxfp4MoeBackend], + requested_activation_key: QuantKey | None, +) -> list[Mxfp4MoeBackend]: + """Pick variants matching ``requested_activation_key``; without one, + prefer BF16 if the list has any, else keep the list as-is so explicit + non-BF16 picks (e.g. the ``_afp8`` aliases) still land.""" + if requested_activation_key is not None: + return [ + b + for b in backends + if _backend_activation_key(b) == requested_activation_key + ] + bf16 = [b for b in backends if _backend_activation_key(b) is None] + return bf16 if bf16 else backends + + def select_mxfp4_moe_backend( config: FusedMoEConfig, activation_key: QuantKey | None = None, @@ -303,8 +391,7 @@ def select_mxfp4_moe_backend( Note: Shape-specific fallbacks may still occur at runtime. """ - # If activation_key is explicitly provided (e.g., W4A8), use it - requested_activation_key = activation_key + requested_activation_key = _resolve_activation_key(activation_key) activation_format = ( mk.FusedMoEActivationFormat.BatchedExperts @@ -312,67 +399,55 @@ def select_mxfp4_moe_backend( else mk.FusedMoEActivationFormat.Standard ) - def _make_log_backend(backend: Mxfp4MoeBackend): - return f"Using '{backend.value}' Mxfp4 MoE backend." - - def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str: - if reason: - return ( - f"Mxfp4 MoE backend '{backend.value}' does not support the " - f"deployment configuration since {reason}." - ) - return ( - f"Mxfp4 MoE backend '{backend.value}' does not support the " - "deployment configuration." - ) - - def _return_or_raise( - backend: Mxfp4MoeBackend, - config: FusedMoEConfig, - weight_key: QuantKey | None, - activation_key: QuantKey | None, - activation_format: mk.FusedMoEActivationFormat, - ) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]: - reason: str | None = None - for k_cls in backend_to_kernel_cls(backend): - supported, reason = k_cls.is_supported_config( - k_cls, config, weight_key, activation_key, activation_format - ) - if supported: - logger.info_once(_make_log_backend(backend)) - return backend, k_cls - raise ValueError(_make_log_unsupported(backend, reason)) - runner_backend = config.moe_backend if runner_backend != "auto": - requested_backend = map_mxfp4_backend(runner_backend) - if ( - activation_format == mk.FusedMoEActivationFormat.BatchedExperts - and requested_backend == Mxfp4MoeBackend.MARLIN - ): - requested_backend = Mxfp4MoeBackend.BATCHED_MARLIN - # Use requested_activation_key if provided, otherwise use backend default - act_key = ( - requested_activation_key - if requested_activation_key is not None - else _backend_activation_key(requested_backend) - ) - return _return_or_raise( - requested_backend, - config, - kMxfp4Static, - act_key, - activation_format, - ) + requested_backends = map_mxfp4_backend(runner_backend) + if activation_format == mk.FusedMoEActivationFormat.BatchedExperts: + requested_backends = [ + Mxfp4MoeBackend.BATCHED_MARLIN if b == Mxfp4MoeBackend.MARLIN else b + for b in requested_backends + ] + candidates = _filter_by_activation(requested_backends, requested_activation_key) + if not candidates: + raise ValueError( + f"moe_backend={runner_backend!r} does not support " + f"activation={requested_activation_key}; supported variants: " + f"{[b.name for b in requested_backends]}" + ) + last_error: Exception | None = None + for requested_backend in candidates: + act_key = ( + requested_activation_key + if requested_activation_key is not None + else _backend_activation_key(requested_backend) + ) + try: + return _return_or_raise( + requested_backend, + config, + kMxfp4Static, + act_key, + activation_format, + ) + except ValueError as e: + last_error = e + assert last_error is not None + raise last_error # Select kernels in order of backend. - AVAILABLE_BACKENDS = _get_priority_backends_for_gpt_oss() + AVAILABLE_BACKENDS = _filter_by_activation( + _get_priority_backends_for_gpt_oss(), requested_activation_key + ) # Handle explicit FlashInfer MXFP4 BF16 configuration. if envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16"): if not envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: - AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16) - AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16) + for _b in ( + Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16, + Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16, + ): + if _b in AVAILABLE_BACKENDS: + AVAILABLE_BACKENDS.remove(_b) else: if current_platform.is_device_capability(90): return _return_or_raise( @@ -481,54 +556,30 @@ def select_deepseek_v4_mxfp4_moe_backend( else mk.FusedMoEActivationFormat.Standard ) - def _make_log_backend(backend: Mxfp4MoeBackend): - return f"Using '{backend.value}' Mxfp4 MoE backend." - - def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str: - if reason: - return ( - f"Mxfp4 MoE backend '{backend.value}' does not support the " - f"deployment configuration since {reason}." - ) - return ( - f"Mxfp4 MoE backend '{backend.value}' does not support the " - "deployment configuration." - ) - - def _return_or_raise( - backend: Mxfp4MoeBackend, - config: FusedMoEConfig, - weight_key: QuantKey | None, - activation_key: QuantKey | None, - activation_format: mk.FusedMoEActivationFormat, - ) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]: - reason: str | None = None - for k_cls in backend_to_kernel_cls(backend): - supported, reason = k_cls.is_supported_config( - k_cls, config, weight_key, activation_key, activation_format - ) - if supported: - logger.info_once(_make_log_backend(backend), scope="local") - return backend, k_cls - raise ValueError(_make_log_unsupported(backend, reason)) - # Honor explicit moe_backend (e.g. "marlin", "triton_unfused") before # falling back to the auto priority list. runner_backend = config.moe_backend if runner_backend != "auto": - requested_backend = map_mxfp4_backend(runner_backend) - if ( - activation_format == mk.FusedMoEActivationFormat.BatchedExperts - and requested_backend == Mxfp4MoeBackend.MARLIN - ): - requested_backend = Mxfp4MoeBackend.BATCHED_MARLIN - return _return_or_raise( - requested_backend, - config, - kMxfp4Static, - _backend_activation_key(requested_backend), - activation_format, - ) + requested_backends = map_mxfp4_backend(runner_backend) + if activation_format == mk.FusedMoEActivationFormat.BatchedExperts: + requested_backends = [ + Mxfp4MoeBackend.BATCHED_MARLIN if b == Mxfp4MoeBackend.MARLIN else b + for b in requested_backends + ] + last_error: Exception | None = None + for requested_backend in requested_backends: + try: + return _return_or_raise( + requested_backend, + config, + kMxfp4Static, + _backend_activation_key(requested_backend), + activation_format, + ) + except ValueError as e: + last_error = e + assert last_error is not None + raise last_error # DeepSeek-V4 on ROCm is more accurate with the unfused Triton MXFP4 path # than the default AITER path. Prefer Triton-unfused for this routing mode, diff --git a/vllm/model_executor/layers/quantization/__init__.py b/vllm/model_executor/layers/quantization/__init__.py index 9f2b4e7020..37534678f0 100644 --- a/vllm/model_executor/layers/quantization/__init__.py +++ b/vllm/model_executor/layers/quantization/__init__.py @@ -35,10 +35,9 @@ QuantizationMethods = Literal[ "deepseek_v4_fp8", "cpu_awq", "online", - # Below are values of the OnlineQuantScheme enum, specified as strings to - # avoid circular import issues. This is here to provide a shortcut where - # the user can specify "LLM(..., quantization='fp8_per_tensor')" as - # shorthand for creating a more complicated online quant config object + # Below are online quant shorthand names (see vllm.config.quantization). + # Listed here as strings to avoid a circular import; kept in sync with + # _ONLINE_SHORTHANDS by the assertion in get_quantization_config(). "fp8_per_tensor", "fp8_per_block", "int8_per_channel_weight_only", @@ -111,7 +110,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]: raise ValueError(f"Invalid quantization method: {quantization}") # lazy import to avoid triggering `torch.compile` too early - from vllm.config.quantization import OnlineQuantScheme + from vllm.config.quantization import _ONLINE_SHORTHANDS from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig from vllm.model_executor.models.deepseek_v4 import DeepseekV4FP8Config @@ -171,16 +170,15 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]: "online": OnlineQuantizationConfig, } - # Below are values of the OnlineQuantScheme enum. This is here to provide - # a shortcut where the user can specify - # "LLM(..., quantization='fp8_per_tensor')" as shorthand for creating a - # more complicated online quant config object - for scheme in OnlineQuantScheme: - assert scheme.value not in method_to_config, ( - f"Online quant scheme {scheme.value!r} conflicts with an " + # Register online shorthands as quantization methods so the user can + # specify "LLM(..., quantization='fp8_per_tensor')" as shorthand for + # creating a more complicated online quant config object. + for shorthand in _ONLINE_SHORTHANDS: + assert shorthand not in method_to_config, ( + f"Online quant shorthand {shorthand!r} conflicts with an " f"existing quantization method" ) - method_to_config[scheme.value] = OnlineQuantizationConfig + method_to_config[shorthand] = OnlineQuantizationConfig # Update the `method_to_config` with customized quantization methods. method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG) diff --git a/vllm/model_executor/layers/quantization/online/base.py b/vllm/model_executor/layers/quantization/online/base.py index 21230177aa..bf166b1818 100644 --- a/vllm/model_executor/layers/quantization/online/base.py +++ b/vllm/model_executor/layers/quantization/online/base.py @@ -5,10 +5,7 @@ from typing import Any import torch -from vllm.config.quantization import ( - OnlineQuantizationConfigArgs, - OnlineQuantScheme, -) +from vllm.config.quantization import QuantizationConfigArgs, QuantSpec from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( RoutedExperts, @@ -41,31 +38,50 @@ from vllm.model_executor.layers.quantization.online.mxfp8 import ( Mxfp8OnlineLinearMethod, Mxfp8OnlineMoEMethod, ) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kFp8Static128BlockSym, + kFp8StaticTensorSym, + kInt8StaticChannelSym, + kMxfp8Dynamic, +) logger = init_logger(__name__) +# Online dispatch tables, keyed by the QuantSpec.weight QuantKey. The +# corresponding method class handles the activation choice via its +# `supported_activation_quant` set. +_ONLINE_LINEAR_METHODS: dict[QuantKey, type] = { + kFp8StaticTensorSym: Fp8PerTensorOnlineLinearMethod, + kFp8Static128BlockSym: Fp8PerBlockOnlineLinearMethod, + kMxfp8Dynamic: Mxfp8OnlineLinearMethod, +} + +_ONLINE_MOE_METHODS: dict[QuantKey, type] = { + kFp8StaticTensorSym: Fp8PerTensorOnlineMoEMethod, + kFp8Static128BlockSym: Fp8PerBlockOnlineMoEMethod, + kMxfp8Dynamic: Mxfp8OnlineMoEMethod, + kInt8StaticChannelSym: Int8OnlineMoEMethod, +} + + class OnlineQuantizationConfig(QuantizationConfig): - """Model-level config class for online quantization (quantize fp16/bf16 weights + """Model-level config for online quantization (quantize fp16/bf16 weights during model loading, without requiring a pre-quantized checkpoint).""" def __init__( self, - args: OnlineQuantizationConfigArgs, + args: QuantizationConfigArgs, ) -> None: super().__init__() - if ( - args.global_scheme is None - and args.linear_scheme_override is None - and args.moe_scheme_override is None - ): + if args.linear is None and args.moe is None: raise ValueError( "OnlineQuantizationConfig requires at least one of " - "global_scheme, linear_scheme_override, or " - "moe_scheme_override to be set." + "quantization_config.linear or quantization_config.moe " + "to be set." ) self.args = args - self.quant_scheme = args.global_scheme self.ignored_layers: list[str] = args.ignore @classmethod @@ -94,6 +110,33 @@ class OnlineQuantizationConfig(QuantizationConfig): "quantization='fp8_per_tensor'/'fp8_per_block' instead." ) + def _dispatch( + self, + spec: QuantSpec | None, + table: dict[QuantKey, type], + layer: torch.nn.Module, + ) -> "QuantizeMethodBase | None": + if spec is None or spec.weight is None: + return None + cls = table.get(spec.weight) + if cls is None: + raise ValueError( + f"online quantization for {type(layer).__name__} with " + f"weight={spec.weight} is not supported; supported weight " + f"keys: {sorted(str(k) for k in table)}" + ) + # Online method classes pick their own activation format internally. + # Per-class activation overrides are not yet wired through; reject + # explicit overrides until the relevant method class opts in. + if spec.activation is not None: + raise ValueError( + f"activation override (activation={spec.activation}) is not " + f"yet supported for online {cls.__name__}" + ) + if isinstance(layer, RoutedExperts): + return cls(layer=layer) + return cls() + def get_quant_method( self, layer: torch.nn.Module, prefix: str ) -> "QuantizeMethodBase | None": @@ -104,20 +147,8 @@ class OnlineQuantizationConfig(QuantizationConfig): fused_mapping=self.packed_modules_mapping, ): return UnquantizedLinearMethod() - - linear_scheme = self.args.linear_scheme_override or self.args.global_scheme - if linear_scheme == OnlineQuantScheme.INT8_PER_CHANNEL_WEIGHT_ONLY: - logger.warning_once( - "INT8 online quantization only quantizes MoE expert " - "weights. linear layers remain in full precision." - ) - return UnquantizedLinearMethod() - elif linear_scheme == OnlineQuantScheme.FP8_PER_BLOCK: - return Fp8PerBlockOnlineLinearMethod() - elif linear_scheme == OnlineQuantScheme.MXFP8: - return Mxfp8OnlineLinearMethod() - else: - return Fp8PerTensorOnlineLinearMethod() + method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer) + return method if method is not None else UnquantizedLinearMethod() elif isinstance(layer, RoutedExperts): if should_ignore_layer( prefix, @@ -125,14 +156,10 @@ class OnlineQuantizationConfig(QuantizationConfig): fused_mapping=self.packed_modules_mapping, ): return UnquantizedFusedMoEMethod(layer.moe_config) - - moe_scheme = self.args.moe_scheme_override or self.args.global_scheme - if moe_scheme == OnlineQuantScheme.INT8_PER_CHANNEL_WEIGHT_ONLY: - return Int8OnlineMoEMethod(layer=layer) - elif moe_scheme == OnlineQuantScheme.FP8_PER_BLOCK: - return Fp8PerBlockOnlineMoEMethod(layer=layer) - elif moe_scheme == OnlineQuantScheme.MXFP8: - return Mxfp8OnlineMoEMethod(layer=layer) - else: - return Fp8PerTensorOnlineMoEMethod(layer=layer) + method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer) + return ( + method + if method is not None + else UnquantizedFusedMoEMethod(layer.moe_config) + ) return None diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index c418c29a8c..de15dc3d53 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -300,12 +300,10 @@ def get_quant_config( ) if hf_quant_config is not None: - if model_config.quantization_config is not None: - raise ValueError( - "Setting `quantization_config` for online " - "quantization when the model checkpoint already " - "has a `quantization_config` is not supported" - ) + # `model_config.quantization_config` may be set alongside a checkpoint + # quant config: the checkpoint determines `quant_cls`, and the user's + # QuantizationConfigArgs is consulted by individual quant methods + # (e.g. for activation overrides via the MXFP4 oracle). # For modelopt_mixed, config.json's quantization_config may or may # not contain the per-layer quantized_layers map. Newer checkpoints @@ -330,12 +328,6 @@ def get_quant_config( quantization_config_file = hf_overrides.get("quantization_config_file", None) if quantization_config_file is not None: if hasattr(quant_cls, "from_config_file"): - if model_config.quantization_config is not None: - raise ValueError( - "Setting `quantization_config` for online " - "quantization when the model checkpoint already " - "has a `quantization_config` is not supported" - ) return quant_cls.from_config_file(quantization_config_file) else: raise NotImplementedError( @@ -346,12 +338,6 @@ def get_quant_config( quantization_config_json = hf_overrides.get("quantization_config_dict_json", None) if quantization_config_json is not None: if hasattr(quant_cls, "from_config_dict_json"): - if model_config.quantization_config is not None: - raise ValueError( - "Setting `quantization_config` for online " - "quantization when the model checkpoint already " - "has a `quantization_config` is not supported" - ) return quant_cls.from_config_dict_json(quantization_config_json) else: raise NotImplementedError( @@ -360,17 +346,15 @@ def get_quant_config( f"{quant_cls}" ) - # Online quantization doesn't read from checkpoint configs — it quantizes + # Online quantization doesn't read from checkpoint configs - it quantizes # fp16/bf16 weights on the fly during loading. if model_config.quantization_config is not None: - from vllm.config.quantization import OnlineQuantizationConfigArgs + from vllm.config.quantization import QuantizationConfigArgs from vllm.model_executor.layers.quantization.online.base import ( OnlineQuantizationConfig, ) - assert isinstance( - model_config.quantization_config, OnlineQuantizationConfigArgs - ) + assert isinstance(model_config.quantization_config, QuantizationConfigArgs) return OnlineQuantizationConfig(args=model_config.quantization_config) # Inflight BNB quantization