From 5690b02c03832a4ac3af231d3ecebe649c188095 Mon Sep 17 00:00:00 2001 From: fxmarty-amd Date: Fri, 4 Sep 2026 20:46:06 +0200 Subject: [PATCH] [Quantization] Support online quantization with partially pre-quantized checkpoints (#51392) Signed-off-by: Felix Marty Signed-off-by: mgoin Co-authored-by: Codex Co-authored-by: mgoin --- docs/features/quantization/online.md | 18 + .../model_executor/test_eagle_quantization.py | 1 + tests/quantization/test_config_utils.py | 2 +- tests/quantization/test_online.py | 566 ++++++++++++++++-- tests/quantization/test_online_mxfp4.py | 2 +- .../test_quantization_config_args.py | 19 +- vllm/config/quantization.py | 26 +- .../layers/attention/attention.py | 11 +- .../layers/attention/mla_attention.py | 7 +- .../layers/fused_moe/routed_experts.py | 3 +- vllm/model_executor/layers/linear.py | 3 +- .../layers/quantization/__init__.py | 6 +- .../layers/quantization/base_config.py | 50 +- .../layers/quantization/experts_int8.py | 2 +- .../model_executor/layers/quantization/fp8.py | 2 +- .../layers/quantization/online/base.py | 166 +++-- .../layers/quantization/online/fp8.py | 25 +- .../layers/quantization/online/int8.py | 5 +- .../layers/quantization/online/mxfp4.py | 9 +- .../layers/quantization/online/mxfp8.py | 10 +- .../layers/quantization/online/nvfp4.py | 9 +- .../layers/vocab_parallel_embedding.py | 3 +- .../model_loader/base_loader.py | 5 + .../model_loader/weight_utils.py | 64 +- vllm/models/kimi_k3/nvidia/mla.py | 7 +- 25 files changed, 846 insertions(+), 175 deletions(-) diff --git a/docs/features/quantization/online.md b/docs/features/quantization/online.md index 342205d000..d3a4edd271 100644 --- a/docs/features/quantization/online.md +++ b/docs/features/quantization/online.md @@ -99,6 +99,24 @@ vllm serve openai/gpt-oss-20b --quantization-config.moe.activation mxfp8 Combine with `--moe-backend` to pin a specific kernel family. +### Online quantization on unquantized layers from partially-quantized checkpoints + +Online quantization can be used on already quantized checkpoints independently of their original `quant_method` (`modelopt`, `compressed-tensors`, `quark`, etc.), for layers that are left unquantized in the original checkpoint. + +The checkpoint `quant_method` remains responsible for its quantized layers, while the selected unquantized layers use the requested online method. + +For example: + +```bash +vllm serve amd/Qwen3.5-35B-A3B-MXFP4 \ + --quantization-config.linear mxfp8 +``` + +adds MXFP8 quantization to the dense linear layers of a Quark checkpoint where only MOE experts are quantized. + +!!! info + `quantization_config.ignore` is an online-only exclusion: the original `quant_method` relies solely on its own ignore implementation and on the ignored layers specified in `config.json`. + ### Separate Schemes for Dense and MoE Layers You can apply different quantization schemes to dense linear layers and MoE expert layers via the `linear` and `moe` fields. Each accepts either a full spec dict, or a bare string naming an online shorthand (e.g. `"fp8_per_block"`) or weight format (e.g. `"fp8_per_block_static"`); fields not set fall back to the shorthand defaults. diff --git a/tests/model_executor/test_eagle_quantization.py b/tests/model_executor/test_eagle_quantization.py index 72c189d833..a9622b3261 100644 --- a/tests/model_executor/test_eagle_quantization.py +++ b/tests/model_executor/test_eagle_quantization.py @@ -83,6 +83,7 @@ def test_fc_layer_quant_config_usage(default_vllm_config, dist_init, device) -> assert fc_no_quant.output_size == output_size mock_quant_config = Mock() + mock_quant_config.online_quantization_config = None fc_with_quant = ReplicatedLinear( input_size=input_size, output_size=output_size, diff --git a/tests/quantization/test_config_utils.py b/tests/quantization/test_config_utils.py index cdab0fbd45..6a0c27748f 100644 --- a/tests/quantization/test_config_utils.py +++ b/tests/quantization/test_config_utils.py @@ -58,7 +58,7 @@ def test_online_ignore_supports_fnmatch_patterns(): ) with pytest.raises(ValueError, match="matches both quantization_config.ignore"): - config._dispatch_target(layer_name, Mock()) + config.resolve_quant_method_cls(Mock(), layer_name) @pytest.mark.parametrize( diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index 4903878bd6..fbd5cbc563 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -2,8 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Tests online quantization.""" +import json +from pathlib import Path from types import SimpleNamespace -from typing import cast +from typing import Any, cast from unittest.mock import Mock import pytest @@ -18,18 +20,39 @@ from tests.quantization.utils import ( from vllm import _custom_ops as ops from vllm._aiter_ops import rocm_aiter_ops from vllm._custom_ops import scaled_fp4_quant -from vllm.config import ModelConfig +from vllm.config.load import LoadConfig +from vllm.config.model import ModelConfig from vllm.config.quantization import ( QuantizationConfigArgs, resolve_quantization_config, ) +from vllm.config.vllm import VllmConfig from vllm.forward_context import set_forward_context +from vllm.model_executor.kernels.linear.mxfp8.emulation import ( + EmulationMxfp8LinearKernel, +) +from vllm.model_executor.kernels.linear.mxfp8.marlin import ( + MarlinMxfp8LinearKernel, +) from vllm.model_executor.layers.attention import Attention +from vllm.model_executor.layers.fused_moe import FusedMoEFactory from vllm.model_executor.layers.linear import ( ColumnParallelLinear, LinearBase, UnquantizedLinearMethod, ) +from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig +from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501 + CompressedTensorsConfig, + CompressedTensorsLinearMethod, +) +from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501 + CompressedTensorsMoEMethod, +) +from vllm.model_executor.layers.quantization.modelopt import ( + ModelOptFp8Config, + ModelOptMxFp8Config, +) from vllm.model_executor.layers.quantization.online.base import ( OnlineQuantizationConfig, ) @@ -50,19 +73,31 @@ from vllm.model_executor.layers.quantization.online.mxfp4 import ( Mxfp4OnlineLinearMethod, Mxfp4OnlineMoEMethod, ) +from vllm.model_executor.layers.quantization.online.mxfp8 import ( + Mxfp8OnlineLinearMethod, +) from vllm.model_executor.layers.quantization.online.nvfp4 import ( Nvfp4OnlineMoEMethod, _quantize_moe_weight_to_nvfp4, ) +from vllm.model_executor.layers.quantization.quark.quark import ( + QuarkConfig, + QuarkLinearMethod, +) from vllm.model_executor.layers.quantization.utils import quant_utils +from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + MXFP8_VALUE_DTYPE, +) from vllm.model_executor.layers.quantization.utils.quant_utils import ( amax_for_moe_weight_quant, amax_for_tp_weight_quant, + kMxfp8Dynamic, weight_amax, ) from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) +from vllm.model_executor.model_loader import weight_utils from vllm.model_executor.model_loader.base_loader import log_online_quantization from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader from vllm.model_executor.models.granitemoe import ( @@ -83,6 +118,10 @@ else: DEVICE = current_platform.device_type +GRANITE_MODEL_NAME = "ibm-granite/granite-3.0-1b-a400m-base" +PARTIALLY_PREQUANTIZED_MODEL_NAME = ( + "nm-testing/tinysmokeqwen3moe-W4A16-first-only-CTstable" +) def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( @@ -132,15 +171,349 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( assert process_weights.call_count == 2 +def _fully_quantized_quark_config() -> QuarkConfig: + return QuarkConfig( + { + "exclude": [], + "global_quant_config": { + "weight": { + "dtype": "int8", + "qscheme": "per_tensor", + "is_dynamic": False, + "symmetric": True, + }, + "input_tensors": { + "dtype": "int8", + "qscheme": "per_tensor", + "is_dynamic": False, + "symmetric": True, + }, + }, + "layer_quant_config": {}, + "layer_type_quant_config": {}, + } + ) + + +def _fully_quantized_modelopt_config() -> ModelOptFp8Config: + return ModelOptFp8Config( + quant_method="FP8", + is_checkpoint_fp8_serialized=True, + kv_cache_quant_method=None, + exclude_modules=[], + ) + + +def _moe_only_compressed_tensors_config() -> CompressedTensorsConfig: + return CompressedTensorsConfig( + target_scheme_map={"RoutedExperts": {}}, + ignore=[], + quant_format="pack-quantized", + ) + + +def _write_minimal_llama_config( + model_path: Path, quantization_config: dict[str, Any] | None = None +) -> None: + config: dict[str, Any] = { + "architectures": ["LlamaForCausalLM"], + "model_type": "llama", + "hidden_size": 32, + "intermediate_size": 64, + "num_attention_heads": 4, + "num_hidden_layers": 1, + "vocab_size": 32, + "max_position_embeddings": 32, + } + if quantization_config is not None: + config["quantization_config"] = quantization_config + (model_path / "config.json").write_text(json.dumps(config)) + + +@pytest.mark.parametrize( + "checkpoint_config_factory,raises_conflict", + [ + pytest.param(_fully_quantized_quark_config, True, id="quark"), + pytest.param(_fully_quantized_modelopt_config, True, id="modelopt"), + pytest.param( + _moe_only_compressed_tensors_config, + False, + id="compressed_tensors", + ), + ], +) +def test_online_prequantized_compatibility( + checkpoint_config_factory, + raises_conflict: bool, + default_vllm_config, + dist_init, +) -> None: + """Online weights replace only layers left unquantized by a checkpoint.""" + default_vllm_config.model_config = ModelConfig() + checkpoint_config = checkpoint_config_factory() + + checkpoint_config.online_quantization_config = OnlineQuantizationConfig( + QuantizationConfigArgs(linear="mxfp8") + ) + config = checkpoint_config + + layer_kwargs = { + "input_size": 32, + "output_size": 32, + "bias": False, + "params_dtype": torch.bfloat16, + "quant_config": config, + "prefix": "model.layers.0.self_attn.o_proj", + "disable_tp": True, + } + + if raises_conflict: + with pytest.raises(ValueError, match="pre-quantized layer"): + ColumnParallelLinear(**layer_kwargs) + else: + layer = ColumnParallelLinear(**layer_kwargs) + assert isinstance(layer.quant_method, Mxfp8OnlineLinearMethod) + + +def test_online_target_rejects_prequantized_layer( + default_vllm_config, dist_init +) -> None: + """A targets match participates in checkpoint compatibility checks.""" + default_vllm_config.model_config = ModelConfig() + prefix = "model.layers.0.self_attn.o_proj" + quant_config = _fully_quantized_quark_config() + quant_config.online_quantization_config = OnlineQuantizationConfig( + QuantizationConfigArgs(targets={prefix: "mxfp8"}) + ) + + with pytest.raises(ValueError, match="pre-quantized layer"): + ColumnParallelLinear( + input_size=32, + output_size=32, + bias=False, + params_dtype=torch.bfloat16, + quant_config=quant_config, + prefix=prefix, + disable_tp=True, + ) + + +def test_online_ignore_keeps_checkpoint_quantization_linear( + default_vllm_config, dist_init, monkeypatch +): + """Ignoring online quantization does not replace a checkpoint method (linear).""" + default_vllm_config.model_config = ModelConfig() + quant_config = _fully_quantized_quark_config() + prefix = "model.layers.0.self_attn.o_proj" + quant_config.online_quantization_config = OnlineQuantizationConfig( + QuantizationConfigArgs(linear="mxfp8", ignore=[prefix]) + ) + monkeypatch.setattr( + quant_config.online_quantization_config, + "get_quant_method", + lambda *args: pytest.fail("online method should not be constructed"), + ) + + layer = ColumnParallelLinear( + input_size=32, + output_size=32, + bias=False, + params_dtype=torch.bfloat16, + quant_config=quant_config, + prefix=prefix, + disable_tp=True, + ) + + assert isinstance(layer.quant_method, QuarkLinearMethod) + + +def test_online_quantization_rejects_prequantized_moe( + default_vllm_config, dist_init +) -> None: + """Online linear and MoE quantization reject a pre-quantized MoE layer.""" + default_vllm_config.model_config = ModelConfig() + prefix = "model.layers.0.mlp.experts" + quant_config = _fully_quantized_quark_config() + quant_config.online_quantization_config = OnlineQuantizationConfig( + QuantizationConfigArgs(linear="mxfp4", moe="mxfp4") + ) + + with pytest.raises(ValueError, match="pre-quantized layer"): + FusedMoEFactory( + num_experts=4, + top_k=2, + hidden_size=32, + intermediate_size=64, + params_dtype=torch.bfloat16, + quant_config=quant_config, + prefix=prefix, + ) + + +def test_activation_only_override_applies_to_checkpoint_method( + default_vllm_config, tmp_path +) -> None: + """Activation-only overrides remain available to the checkpoint method.""" + _write_minimal_llama_config(tmp_path) + model_config = ModelConfig( + model=str(tmp_path), + quantization="compressed-tensors", + quantization_config=QuantizationConfigArgs(moe={"activation": "mxfp8"}), + hf_overrides={ + "quantization_config": { + "quant_method": "compressed-tensors", + "format": "pack-quantized", + "config_groups": {}, + } + }, + ) + + result = weight_utils.get_quant_config(model_config, LoadConfig()) + + assert isinstance(result, CompressedTensorsConfig) + assert isinstance(result.online_quantization_config, OnlineQuantizationConfig) + + default_vllm_config.model_config = model_config + from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( + _resolve_activation_key, + ) + + assert _resolve_activation_key(None) == kMxfp8Dynamic + + +def test_online_overlay_requires_normalized_quantization_config(tmp_path) -> None: + """The loader rejects a raw config dict instead of silently ignoring it.""" + _write_minimal_llama_config(tmp_path) + model_config = ModelConfig( + model=str(tmp_path), + quantization="compressed-tensors", + quantization_config={"linear": "mxfp8"}, + hf_overrides={ + "quantization_config": { + "quant_method": "compressed-tensors", + "format": "pack-quantized", + "config_groups": {}, + } + }, + ) + + # assert isinstance(online_args, QuantizationConfigArgs) + # in maybe_compose_online_quantization + with pytest.raises(AssertionError): + weight_utils.get_quant_config(model_config, LoadConfig()) + + +def test_online_overlay_loads_checkpoint_config_file(tmp_path) -> None: + """An online overlay must not bypass checkpoint config file loading.""" + _write_minimal_llama_config(tmp_path) + (tmp_path / "quantize_config.json").write_text( + '{"w_bit": 4, "q_group_size": 128, "zero_point": true}' + ) + model_config = ModelConfig( + model=str(tmp_path), + quantization="awq", + quantization_config=QuantizationConfigArgs(linear="mxfp8"), + ) + + result = weight_utils.get_quant_config(model_config, LoadConfig()) + + assert isinstance(result, AutoAWQConfig) + assert result.weight_bits == 4 + assert result.group_size == 128 + assert result.zero_point + assert isinstance(result.online_quantization_config, OnlineQuantizationConfig) + assert result.online_quantization_config.args is model_config.quantization_config + + +@pytest.mark.parametrize( + "quantization,checkpoint_quantization_config,is_config_file", + [ + pytest.param("fp8_per_block", None, False, id="fp8-per-block-online"), + pytest.param("mxfp4", None, False, id="mxfp4-online"), + pytest.param("mxfp8", None, False, id="mxfp8-online"), + pytest.param( + "mxfp8", + {"quant_method": "mxfp8", "ignored_layers": []}, + False, + id="mxfp8-inline-checkpoint-config", + ), + pytest.param( + "mxfp8", + {"quant_method": "mxfp8", "ignored_layers": []}, + True, + id="mxfp8-checkpoint-config-file", + ), + ], +) +def test_online_shorthand_selects_checkpoint_or_online_config( + tmp_path, quantization, checkpoint_quantization_config, is_config_file +) -> None: + """Online shorthands defer to checkpoint metadata when available.""" + + if is_config_file: + (tmp_path / "hf_quant_config.json").write_text( + '{"quant_method": "mxfp8", "ignored_layers": []}' + ) + + _write_minimal_llama_config( + tmp_path, + None if is_config_file else checkpoint_quantization_config, + ) + model_config = ModelConfig(model=str(tmp_path), quantization=quantization) + + result = weight_utils.get_quant_config(model_config, LoadConfig()) + + if checkpoint_quantization_config is not None: + assert isinstance(result, ModelOptMxFp8Config) + assert result.online_quantization_config is None + else: + assert isinstance(result, OnlineQuantizationConfig) + assert result.online_quantization_config is None + + +def test_log_online_quantization_for_composable_config(monkeypatch) -> None: + """Composable configs log their nested online quantization results.""" + online_config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8")) + online_config.quantized_layers = { + "model.layers.0.self_attn.o_proj": ("linear", "mxfp8", None), + "model.layers.1.self_attn.o_proj": ("linear", "mxfp8", None), + } + vllm_config = VllmConfig(quant_config=online_config) + log_args = [] + monkeypatch.setattr( + "vllm.model_executor.model_loader.base_loader.logger.info", + lambda *args: log_args.append(args), + ) + + log_online_quantization(vllm_config) + + assert log_args == [ + ( + "Quantized %d layers of types: %s", + 2, + "; ".join(online_config.quantized_layer_summaries), + ) + ] + + +def test_checkpoint_quantization_rejects_online_shorthand(tmp_path) -> None: + """A checkpoint quant method cannot be replaced by an online shorthand.""" + _write_minimal_llama_config(tmp_path, {"quant_method": "mxfp4"}) + + with pytest.raises(ValueError, match="does not match the quantization"): + ModelConfig(model=str(tmp_path), quantization="fp8_per_channel") + + @pytest.mark.skipif( not is_quant_method_supported("fp8"), reason="FP8 is not supported on this GPU type.", ) @pytest.mark.parametrize( - "quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls", + ("model_name,quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls"), [ # simple case - quantization='fp8_per_tensor' ( + GRANITE_MODEL_NAME, "fp8_per_tensor", None, Fp8PerTensorOnlineLinearMethod, @@ -148,12 +521,14 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( ), # simple case - quantization='fp8_per_block' ( + GRANITE_MODEL_NAME, "fp8_per_block", None, Fp8PerBlockOnlineLinearMethod, Fp8PerBlockOnlineMoEMethod, ), ( + GRANITE_MODEL_NAME, "fp8_per_channel", None, Fp8PtpcOnlineLinearMethod, @@ -161,6 +536,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( ), # quantization='online' with per-layer-kind overrides ( + GRANITE_MODEL_NAME, "online", { "linear": "fp8_per_block", @@ -171,6 +547,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( ), # quantization='online' with per-layer target patterns ( + GRANITE_MODEL_NAME, "online", { "targets": { @@ -183,6 +560,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( ), # ignore with direct layer name ( + GRANITE_MODEL_NAME, "fp8_per_tensor", # qkv_proj is fused from q_proj/k_proj/v_proj. The shard regex # remains supported alongside direct fused-name regexes. @@ -191,11 +569,20 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( Fp8PerTensorOnlineMoEMethod, ), ( + GRANITE_MODEL_NAME, "mxfp4", None, Mxfp4OnlineLinearMethod, Mxfp4OnlineMoEMethod, ), + pytest.param( + PARTIALLY_PREQUANTIZED_MODEL_NAME, + None, + {"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}}, + Mxfp8OnlineLinearMethod, + CompressedTensorsMoEMethod, + id="partially_prequantized_checkpoint", + ), ], ids=[ "fp8_per_tensor", @@ -205,13 +592,15 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( "targets", "ignore", "mxfp4", + "partially_prequantized_checkpoint", ], ) @pytest.mark.parametrize( "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False] ) def test_online_quantization( - quant_scheme: str, + model_name: str, + quant_scheme: str | None, online_quant_args: dict | None, expected_linear_cls, expected_moe_cls, @@ -243,38 +632,74 @@ def test_online_quantization( ): pytest.skip(f"Skip test for online {quant_scheme} on XPU platform.") - model_name = "ibm-granite/granite-3.0-1b-a400m-base" - model, vllm_config = load_model_without_vllm_runner( - model_name, - dtype="bfloat16", - quantization=quant_scheme, - model_config_kwargs={ - "quantization_config": resolve_quantization_config( - quant_scheme, online_quant_args - ), - "hf_overrides": { - "num_hidden_layers": 3, - "vocab_size": 256, - "hidden_size": 256, - "intermediate_size": 512, - "num_attention_heads": 4, - "num_key_value_heads": 4, - "max_position_embeddings": 64, - "num_local_experts": 4, - "num_experts_per_tok": 2, + if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME: + model, vllm_config = load_model_without_vllm_runner( + model_name, + model_config_kwargs={ + "quantization_config": resolve_quantization_config( + None, online_quant_args + ) }, - }, - model_loader_cls=DummyModelLoader, - ) + model_loader_cls=DummyModelLoader, + ) + assert isinstance( + model.model.layers[1].self_attn.o_proj.quant_method, + expected_linear_cls, + ) + assert isinstance( + model.model.layers[0].mlp.experts._quant_method, + expected_moe_cls, + ) + assert isinstance( + model.model.layers[0].self_attn.o_proj.quant_method, + CompressedTensorsLinearMethod, + ) + else: + model, vllm_config = load_model_without_vllm_runner( + model_name, + dtype="bfloat16", + quantization=quant_scheme, + model_config_kwargs={ + "quantization_config": resolve_quantization_config( + quant_scheme, online_quant_args + ), + "hf_overrides": { + "num_hidden_layers": 3, + "vocab_size": 256, + "hidden_size": 256, + "intermediate_size": 512, + "num_attention_heads": 4, + "num_key_value_heads": 4, + "max_position_embeddings": 64, + "num_local_experts": 4, + "num_experts_per_tok": 2, + }, + }, + model_loader_cls=DummyModelLoader, + ) monkeypatch.setattr(Attention, "forward", lambda _, q, k, v: q.contiguous()) - o_proj = model.model.layers[0].self_attn.o_proj - moe = model.model.layers[0].block_sparse_moe.experts + if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME: + o_proj = model.model.layers[1].self_attn.o_proj + moe = model.model.layers[0].mlp.experts + else: + o_proj = model.model.layers[0].self_attn.o_proj + moe = model.model.layers[0].block_sparse_moe.experts assert isinstance(o_proj.quant_method, expected_linear_cls) assert isinstance(moe._quant_method, expected_moe_cls) - if quant_scheme == "mxfp4": + if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance( + o_proj.quant_method.kernel, MarlinMxfp8LinearKernel + ): + assert o_proj.weight.dtype == torch.int32 + elif model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance( + o_proj.quant_method.kernel, EmulationMxfp8LinearKernel + ): + assert o_proj.weight.dtype == torch.bfloat16 + elif model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME: + assert o_proj.weight.dtype == MXFP8_VALUE_DTYPE + elif quant_scheme == "mxfp4": assert o_proj.weight.dtype == torch.uint8 elif current_platform.is_cuda() or current_platform.is_xpu(): assert o_proj.weight.dtype == torch.float8_e4m3fn @@ -301,7 +726,11 @@ def test_online_quantization( layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod ) - if isinstance(online_quant_args, dict) and "targets" in online_quant_args: + if ( + model_name == GRANITE_MODEL_NAME + and isinstance(online_quant_args, dict) + and "targets" in online_quant_args + ): for layer in model.model.layers: assert isinstance( layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod @@ -319,8 +748,13 @@ def test_online_quantization( not is_quant_method_supported("fp8"), reason="FP8 is not supported on this GPU type.", ) -def test_online_quantization_loads_real_weights(vllm_runner, monkeypatch) -> None: - """Verify online quantization loads a Granite-MoE checkpoint end to end.""" +@pytest.mark.parametrize( + "model_name", [GRANITE_MODEL_NAME, PARTIALLY_PREQUANTIZED_MODEL_NAME] +) +def test_online_quantization_loads_real_weights( + model_name, vllm_runner, monkeypatch +) -> None: + """Verify online quantization loads real checkpoints end to end.""" monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") @@ -336,26 +770,52 @@ def test_online_quantization_loads_real_weights(vllm_runner, monkeypatch) -> Non monkeypatch.setattr(GraniteMoeModel, "load_weights", load_weights) - with vllm_runner( - "ibm-granite/granite-3.0-1b-a400m-base", - quantization="fp8_per_tensor", - dtype="bfloat16", - enforce_eager=True, - hf_overrides={"num_hidden_layers": 3}, - max_model_len=16, - max_num_seqs=1, - ) as llm: + if model_name == GRANITE_MODEL_NAME: + runner_kwargs = { + "quantization": "fp8_per_tensor", + "dtype": "bfloat16", + "enforce_eager": True, + "hf_overrides": {"num_hidden_layers": 3}, + "max_model_len": 16, + "max_num_seqs": 1, + } + else: + runner_kwargs = { + "quantization_config": { + "targets": {"model.layers.1.self_attn.o_proj": "mxfp8"} + }, + "enforce_eager": True, + } - def check_model(model): - layer = model.model.layers[0] - assert isinstance( - layer.self_attn.o_proj.quant_method, - Fp8PerTensorOnlineLinearMethod, - ) - assert isinstance( - layer.block_sparse_moe.experts._quant_method, - Fp8PerTensorOnlineMoEMethod, - ) + with vllm_runner(model_name, **runner_kwargs) as llm: + if model_name == GRANITE_MODEL_NAME: + + def check_model(model): + layer = model.model.layers[0] + assert isinstance( + layer.self_attn.o_proj.quant_method, + Fp8PerTensorOnlineLinearMethod, + ) + assert isinstance( + layer.block_sparse_moe.experts._quant_method, + Fp8PerTensorOnlineMoEMethod, + ) + + else: + + def check_model(model): + assert isinstance( + model.model.layers[1].self_attn.o_proj.quant_method, + Mxfp8OnlineLinearMethod, + ) + assert isinstance( + model.model.layers[0].self_attn.o_proj.quant_method, + CompressedTensorsLinearMethod, + ) + assert isinstance( + model.model.layers[0].mlp.experts._quant_method, + CompressedTensorsMoEMethod, + ) llm.apply_model(check_model) outputs = llm.generate_greedy(["Hello my name is"], max_tokens=1) @@ -457,8 +917,8 @@ def test_online_quantization_targets_ignore_collision() -> None: ) ) with pytest.raises(ValueError, match="matches both quantization_config.ignore"): - config._dispatch_target( - "model.layers.0.self_attn.o_proj", Mock(spec=LinearBase) + config.resolve_quant_method_cls( + Mock(spec=LinearBase), "model.layers.0.self_attn.o_proj" ) diff --git a/tests/quantization/test_online_mxfp4.py b/tests/quantization/test_online_mxfp4.py index f4112875d4..c435790bce 100644 --- a/tests/quantization/test_online_mxfp4.py +++ b/tests/quantization/test_online_mxfp4.py @@ -354,7 +354,7 @@ def test_online_mxfp4_moe_matches_quark( weight_quant_key=kMxfp4Static, activation_quant_key=kMxfp4Dynamic, ) - online_method = Mxfp4OnlineMoEMethod(layer=online_layer) + online_method = Mxfp4OnlineMoEMethod(moe=online_layer.moe_config) # `RoutedExperts.__init__` applies this round-up in production; these # layers are built without a quant config, so it is applied explicitly. diff --git a/tests/quantization/test_quantization_config_args.py b/tests/quantization/test_quantization_config_args.py index 1b67c18caa..0bf203ac45 100644 --- a/tests/quantization/test_quantization_config_args.py +++ b/tests/quantization/test_quantization_config_args.py @@ -95,6 +95,12 @@ def test_resolve_shorthand_only_populates_both_slots(): assert args.moe == QuantSpec(weight=kFp8Static128BlockSym) +@pytest.mark.parametrize("quantization", ["mxfp4", "mxfp8"]) +def test_resolve_colliding_shorthand_is_deferred(quantization: str): + """Checkpoint metadata determines whether an MXFP shorthand is online.""" + assert resolve_quantization_config(quantization, None) is None + + def test_resolve_int8_shorthand_leaves_linear_unset(): # int8_per_channel_weight_only is MoE-only; linear stays None so that # OnlineQuantizationConfig leaves Linear layers in full precision. @@ -122,12 +128,9 @@ def test_resolve_merges_explicit_over_shorthand(): assert args.moe == QuantSpec(weight=kFp8StaticTensorSym) -def test_resolve_rejects_quantization_config_with_non_shorthand_quant(): - # If --quantization names something other than an online shorthand, - # quantization_config is not allowed via this path (checkpoint quant - # paths read it directly off ModelConfig instead). - with pytest.raises(ValueError, match="quantization_config is only supported"): - resolve_quantization_config("gptq", {"linear": "fp8_per_block"}) +def test_resolve_quantization_config_with_checkpoint_quantization(): + args = resolve_quantization_config("gptq", {"linear": "fp8_per_block"}) + assert args == QuantizationConfigArgs(linear="fp8_per_block") # ---- QUANT_KEY_NAMES coverage -------------------------------------------- @@ -206,6 +209,6 @@ def test_targets_reject_moe_only_shorthand_for_linear_layer(): ) with pytest.raises(ValueError, match="does not define a QuantSpec"): - config._dispatch_target( - "model.layers.0.self_attn.o_proj", Mock(spec=LinearBase) + config.resolve_quant_method_cls( + Mock(spec=LinearBase), "model.layers.0.self_attn.o_proj" ) diff --git a/vllm/config/quantization.py b/vllm/config/quantization.py index 2d78e37a79..7d32a17308 100644 --- a/vllm/config/quantization.py +++ b/vllm/config/quantization.py @@ -225,6 +225,10 @@ ONLINE_QUANT_SHORTHAND_NAMES: tuple[str, ...] = ( "online", ) +# These names are also checkpoint quantization methods. Their online configs +# are resolved only when checkpoint quantization metadata is absent. +_DEFERRED_ONLINE_SHORTHANDS = frozenset(("mxfp4", "mxfp8")) + def resolve_quantization_config( quantization: str | None, @@ -239,17 +243,25 @@ def resolve_quantization_config( take precedence over the shorthand. """ if quantization is not None and quantization not in ONLINE_QUANT_SHORTHAND_NAMES: - if quantization_config is not None: - raise ValueError( - f"quantization_config is only supported when quantization is " - f"one of {sorted(ONLINE_QUANT_SHORTHAND_NAMES)}, " - f"got quantization={quantization!r}" - ) - return None + # Pre-quantized checkpoints can be composed with online quantization + # for layers that the base quant_method leaves unquantized. The + # checkpoint quant_method remains the primary quantization method; composition + # is performed after its config has been loaded. + if quantization_config is None: + return None + + # `quantization_config` may hold both: + # 1. Base quantization method activation key override, + # 2. online quantization config to apply on top of the base quant_method. + if isinstance(quantization_config, dict): + return QuantizationConfigArgs(**quantization_config) + return quantization_config base = _ONLINE_SHORTHANDS.get(quantization) if quantization else None if quantization_config is None: + if quantization in _DEFERRED_ONLINE_SHORTHANDS: + return None return base if isinstance(quantization_config, dict): diff --git a/vllm/model_executor/layers/attention/attention.py b/vllm/model_executor/layers/attention/attention.py index ff0c7a33af..c6ba8ea712 100644 --- a/vllm/model_executor/layers/attention/attention.py +++ b/vllm/model_executor/layers/attention/attention.py @@ -19,7 +19,10 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.linear import ( UnquantizedLinearMethod, ) -from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization import ( + QuantizationConfig, + resolve_quant_method, +) from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod @@ -183,7 +186,9 @@ def _init_kv_cache_quant( layer._o_scale_float = None quant_method = ( - quant_config.get_quant_method(layer, prefix=prefix) if quant_config else None + resolve_quant_method(quant_config, layer, prefix=prefix) + if quant_config + else None ) # See [Note: Register q/k/v/prob scales in state dict] @@ -584,7 +589,7 @@ class Attention(nn.Module, AttentionLayerBase): # as the default value. See [Note: Register q/k/v/prob scales in state dict] # for more details. quant_method = ( - self.quant_config.get_quant_method(self, prefix=self.layer_name) + resolve_quant_method(self.quant_config, self, prefix=self.layer_name) if self.quant_config else None ) diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index f0cdd80374..876a625894 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -246,7 +246,10 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.linear import ( ColumnParallelLinear, ) -from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization import ( + QuantizationConfig, + resolve_quant_method, +) from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8 from vllm.model_executor.layers.quantization.utils.quant_utils import ( GroupShape, @@ -1255,7 +1258,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): # as the default value. See [Note: Register q/k/v/prob scales in state dict] # for more details. quant_method = ( - self.quant_config.get_quant_method(self, prefix=self.layer_name) + resolve_quant_method(self.quant_config, self, prefix=self.layer_name) if self.quant_config else None ) diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index 59562ed8be..e9a7fc4aa7 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -25,6 +25,7 @@ from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import ( ) from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, + resolve_quant_method, ) if TYPE_CHECKING: @@ -199,7 +200,7 @@ class RoutedExperts(PluggableLayer): """ quant_method = None if quant_config is not None: - quant_method = quant_config.get_quant_method(self, prefix) + quant_method = resolve_quant_method(quant_config, self, prefix) if quant_method is None: quant_method = UnquantizedFusedMoEMethod(moe_config) assert isinstance(quant_method, FusedMoEMethodBase) diff --git a/vllm/model_executor/layers/linear.py b/vllm/model_executor/layers/linear.py index c7a366012a..cd1c0e69d2 100644 --- a/vllm/model_executor/layers/linear.py +++ b/vllm/model_executor/layers/linear.py @@ -24,6 +24,7 @@ from vllm.model_executor.custom_op import PluggableLayer from vllm.model_executor.determinism.batch_invariant import ( linear_batch_invariant, ) +from vllm.model_executor.layers.quantization import resolve_quant_method from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase, @@ -282,7 +283,7 @@ class LinearBase(PluggableLayer): self.quant_method: QuantizeMethodBase if quant_config is None: self.quant_method = UnquantizedLinearMethod() - elif quant_method := quant_config.get_quant_method(self, prefix=prefix): + elif quant_method := resolve_quant_method(quant_config, self, prefix=prefix): self.quant_method = quant_method else: raise ValueError("All linear layers should support quant method.") diff --git a/vllm/model_executor/layers/quantization/__init__.py b/vllm/model_executor/layers/quantization/__init__.py index 28dc9a688e..a2c65738e2 100644 --- a/vllm/model_executor/layers/quantization/__init__.py +++ b/vllm/model_executor/layers/quantization/__init__.py @@ -4,7 +4,10 @@ from typing import Literal, get_args from vllm.logger import init_logger -from vllm.model_executor.layers.quantization.base_config import QuantizationConfig +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, + resolve_quant_method, +) from vllm.platforms import current_platform logger = init_logger(__name__) @@ -182,6 +185,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]: __all__ = [ "QuantizationConfig", + "resolve_quant_method", "QuantizationMethods", "get_quantization_config", "register_quantization_config", diff --git a/vllm/model_executor/layers/quantization/base_config.py b/vllm/model_executor/layers/quantization/base_config.py index 917d229b17..f83c1ec76f 100644 --- a/vllm/model_executor/layers/quantization/base_config.py +++ b/vllm/model_executor/layers/quantization/base_config.py @@ -12,6 +12,9 @@ from transformers import PretrainedConfig if TYPE_CHECKING: from vllm.model_executor.layers.quantization import QuantizationMethods + from vllm.model_executor.layers.quantization.online.base import ( + OnlineQuantizationConfig, + ) from vllm.model_executor.models.utils import WeightsMapper else: QuantizationMethods = str @@ -105,6 +108,7 @@ class QuantizationConfig(ABC): """Suffixes of quantization parameters that may be present in the checkpoint but not in the model, and should be ignored if unexpected during loading. These are used after remapping, so should be in vLLM format (e.g. .q_scale, not .q.scale).""" + online_quantization_config: "OnlineQuantizationConfig | None" = None def __init__(self): super().__init__() @@ -187,7 +191,8 @@ class QuantizationConfig(ABC): def get_quant_method( self, layer: torch.nn.Module, prefix: str ) -> QuantizeMethodBase | None: - """Get the quantize method to use for the quantized layer. + """Get the quantize method to use for the quantized layer, from the + pre-quantized checkpoint quant_method. Args: layer: The layer for the quant method. @@ -265,3 +270,46 @@ class QuantizationConfig(ABC): # TODO: revision is never passed currently in vllm.py, # but is used in subclasses, should we remove this parameter? pass + + +def resolve_quant_method( + quant_config: QuantizationConfig, layer: torch.nn.Module, prefix: str +) -> QuantizeMethodBase | None: + """Return the checkpoint method with configured online quantization.""" + from vllm.model_executor.layers.fused_moe import RoutedExperts + from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import ( + UnquantizedFusedMoEMethod, + ) + from vllm.model_executor.layers.linear import ( + LinearBase, + UnquantizedLinearMethod, + ) + + base_quant_method = quant_config.get_quant_method(layer, prefix) + if quant_config.online_quantization_config is None: + return base_quant_method + # Online quantization currently supports only LinearBase and RoutedExperts. + # Embeddings and ParallelLMHead retain their checkpoint quantization method. + if not isinstance(layer, (LinearBase, RoutedExperts)): + return base_quant_method + + quant_config.online_quantization_config.packed_modules_mapping = ( + quant_config.packed_modules_mapping + ) + checkpoint_is_quantized = base_quant_method is not None and not isinstance( + base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod) + ) + online_target = quant_config.online_quantization_config.resolve_quant_method_cls( + layer, prefix + ) + if checkpoint_is_quantized: + if online_target is not None: + raise ValueError( + f"Cannot apply requested online quantization {online_target[3]} to " + f"pre-quantized layer {prefix}: {base_quant_method} was already " + "selected by the checkpoint quantization config." + ) + return base_quant_method + if online_target is None: + return base_quant_method + return quant_config.online_quantization_config.get_quant_method(layer, prefix) diff --git a/vllm/model_executor/layers/quantization/experts_int8.py b/vllm/model_executor/layers/quantization/experts_int8.py index 3db2916055..cbde4d9603 100644 --- a/vllm/model_executor/layers/quantization/experts_int8.py +++ b/vllm/model_executor/layers/quantization/experts_int8.py @@ -56,5 +56,5 @@ class ExpertsInt8Config(QuantizationConfig): if isinstance(layer, LinearBase): return UnquantizedLinearMethod() elif isinstance(layer, RoutedExperts): - return Int8OnlineMoEMethod(layer=layer) + return Int8OnlineMoEMethod(moe=layer.moe_config) return None diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 0b057f7fd0..2cf7168830 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -219,7 +219,7 @@ class Fp8Config(QuantizationConfig): Fp8PerTensorOnlineMoEMethod, ) - return Fp8PerTensorOnlineMoEMethod(layer=layer) + return Fp8PerTensorOnlineMoEMethod(moe=layer.moe_config) elif isinstance(layer, Attention): return Fp8KVCacheMethod(self) return None diff --git a/vllm/model_executor/layers/quantization/online/base.py b/vllm/model_executor/layers/quantization/online/base.py index a9f994daef..a16a33eaaa 100644 --- a/vllm/model_executor/layers/quantization/online/base.py +++ b/vllm/model_executor/layers/quantization/online/base.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Mapping +from enum import Enum from types import MappingProxyType from typing import Any @@ -14,6 +15,7 @@ from vllm.config.quantization import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( + FusedMoEMethodBase, RoutedExperts, ) from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import ( @@ -38,6 +40,7 @@ from vllm.model_executor.layers.quantization.online.fp8 import ( Fp8PerTensorOnlineMoEMethod, Fp8PtpcOnlineLinearMethod, Fp8PtpcOnlineMoEMethod, + OnlineLinearBase, ) from vllm.model_executor.layers.quantization.online.int8 import ( Int8OnlineMoEMethod, @@ -71,6 +74,14 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( logger = init_logger(__name__) +class OnlineQuantizationSource(str, Enum): + """Supported online quantization configuration sources.""" + + linear = "linear" # LinearBase + moe = "moe" # RoutedExperts + targets = "targets" + + # Online dispatch tables, keyed by the QuantSpec.weight QuantKey. The # corresponding method class handles the activation choice via its # `supported_activation_quant` set. @@ -200,12 +211,23 @@ class OnlineQuantizationConfig(QuantizationConfig): "quantization='fp8_per_tensor'/'fp8_per_block' instead." ) - def _dispatch( + def _get_method_cls( self, spec: QuantSpec | None, table: dict[QuantKey, type], layer: torch.nn.Module, - ) -> "QuantizeMethodBase | None": + ) -> type | None: + """Resolve the online method class for a layer's quantization spec. + + Args: + spec: Quantization specification to resolve. + table: Mapping from weight quantization keys to method classes. + layer: Layer that will use the resolved method. + + Returns: + The matching method class, or None when ``spec`` has no weight + quantization. + """ if spec is None or spec.weight is None: return None cls = table.get(spec.weight) @@ -223,13 +245,75 @@ class OnlineQuantizationConfig(QuantizationConfig): f"activation override (activation={spec.activation}) is not " f"yet supported for online {cls.__name__}" ) - if isinstance(layer, RoutedExperts): - return cls(layer=layer) - return cls() + return cls - def _dispatch_target( + def resolve_quant_method_cls( + self, layer: torch.nn.Module, prefix: str + ) -> tuple[OnlineQuantizationSource, str, str | None, QuantSpec, type] | None: + """Resolve quantization metadata and method class without instantiating it. + + Args: + layer: Layer for which to resolve online quantization. + prefix: Fully qualified layer name. + + Returns: + A tuple of source, quantization key string, target pattern, spec, + and method class. Returns None when online quantization does not + apply to the layer. + """ + quant_spec: QuantSpec | None + if self.args.targets is not None: + resolved_pattern = self._resolve_targets_quant_method_metadata( + prefix, layer + ) + if resolved_pattern is None: + return None + source, quant_key_str, target_pattern, quant_spec, table = resolved_pattern + else: + if isinstance(layer, LinearBase): + source = OnlineQuantizationSource.linear + quant_spec = self.args.linear + table = _ONLINE_LINEAR_METHODS + elif isinstance(layer, RoutedExperts): + source = OnlineQuantizationSource.moe + quant_spec = self.args.moe + table = _ONLINE_MOE_METHODS + else: + return None + + if should_ignore_layer( + prefix, + ignore=self.ignored_layers, + fused_mapping=self.packed_modules_mapping, + use_fnmatch=True, + ): + return None + quant_key_str = str(quant_spec) + target_pattern = None + + quant_method_cls = self._get_method_cls(quant_spec, table, layer) + if quant_method_cls is None: + return None + assert quant_spec is not None + return source, quant_key_str, target_pattern, quant_spec, quant_method_cls + + def _resolve_targets_quant_method_metadata( self, prefix: str, layer: torch.nn.Module - ) -> "QuantizeMethodBase | None": + ) -> ( + tuple[OnlineQuantizationSource, str, str, QuantSpec, dict[QuantKey, type]] + | None + ): + """Resolve target-pattern quantization metadata and dispatch table. + + Args: + prefix: Fully qualified layer name. + layer: Layer matched against configured target patterns. + + Returns: + A tuple of source, quantization key string, target pattern, spec, + and dispatch table. Returns None when no pattern applies or the + layer is ignored. + """ assert self.args.targets is not None ignored = should_ignore_layer( prefix, @@ -254,7 +338,8 @@ class OnlineQuantizationConfig(QuantizationConfig): f"targets patterns: {matches}. Each layer may match at most " f"one target." ) - quant_key_str = self.args.targets[matches[0]] + target_pattern = matches[0] + quant_key_str = self.args.targets[target_pattern] shorthand = _ONLINE_SHORTHANDS[quant_key_str] if isinstance(layer, LinearBase): quant_spec = shorthand.linear @@ -265,63 +350,44 @@ class OnlineQuantizationConfig(QuantizationConfig): else: raise ValueError( f"Layer {prefix} was matched by quantization_config.targets " - f"({matches[0]}), but online quantization is not supported for " + f"({target_pattern}), but online quantization is not supported for " f"{type(layer).__name__}." ) if quant_spec is None: raise ValueError( - f"targets pattern {matches[0]} = {quant_key_str} does " + f"targets pattern {target_pattern} = {quant_key_str} does " f"not define a QuantSpec for {type(layer).__name__} layers " f"(matched at {prefix})." ) - method = self._dispatch(quant_spec, table, layer) - if method is not None: - self.quantized_layers[prefix] = ("targets", quant_key_str, matches[0]) - return method + return ( + OnlineQuantizationSource.targets, + quant_key_str, + target_pattern, + quant_spec, + table, + ) def get_quant_method( self, layer: torch.nn.Module, prefix: str ) -> "QuantizeMethodBase | None": # `targets` takes precedence over `moe` and `linear` and is exclusive. - if self.args.targets is not None: - method = self._dispatch_target(prefix, layer) - if method is not None: - return method - - if isinstance(layer, LinearBase): - return UnquantizedLinearMethod() + resolved = self.resolve_quant_method_cls(layer, prefix) + if resolved is not None: + source, quant_key_str, target_pattern, _, quant_method_cls = resolved + self.quantized_layers[prefix] = ( + source.value, + quant_key_str, + target_pattern, + ) if isinstance(layer, RoutedExperts): - return UnquantizedFusedMoEMethod(layer.moe_config) - return None + assert issubclass(quant_method_cls, FusedMoEMethodBase) + return quant_method_cls(moe=layer.moe_config) + + assert issubclass(quant_method_cls, OnlineLinearBase) + return quant_method_cls() if isinstance(layer, LinearBase): - if should_ignore_layer( - prefix, - ignore=self.ignored_layers, - fused_mapping=self.packed_modules_mapping, - use_fnmatch=True, - ): - return UnquantizedLinearMethod() - method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer) - if method is not None: - self.quantized_layers[prefix] = ( - "linear", - str(self.args.linear), - None, - ) - return method return UnquantizedLinearMethod() - elif isinstance(layer, RoutedExperts): - if should_ignore_layer( - prefix, - ignore=self.ignored_layers, - fused_mapping=self.packed_modules_mapping, - use_fnmatch=True, - ): - return UnquantizedFusedMoEMethod(layer.moe_config) - method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer) - if method is not None: - self.quantized_layers[prefix] = ("moe", str(self.args.moe), None) - return method + if isinstance(layer, RoutedExperts): return UnquantizedFusedMoEMethod(layer.moe_config) return None diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index a8720b2d23..0234702c8b 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -23,6 +23,7 @@ from vllm.model_executor.kernels.linear.scaled_mm import ( MarlinFP8ScaledMMLinearKernel, ) from vllm.model_executor.layers.fused_moe import RoutedExperts +from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( select_fp8_moe_backend, ) @@ -111,7 +112,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool: return is_row_parallel or (reduces_output_dim and is_column_parallel) -class _Fp8OnlineLinearBase(LinearMethodBase): +class OnlineLinearBase(LinearMethodBase): """Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint weights onto meta device and materializes them just-in-time.""" @@ -155,7 +156,7 @@ class _Fp8OnlineLinearBase(LinearMethodBase): initialize_online_processing(layer) -class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase): +class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase): """Online tensorwise FP8 linear quantization. Loads fp16/bf16 weights and quantizes them per-tensor during loading.""" @@ -256,7 +257,7 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase): return self.fp8_linear.apply_weights(layer, x, bias) -class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase): +class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase): """Online blockwise FP8 linear quantization. Loads fp16/bf16 weights and quantizes them per-block during loading.""" @@ -336,7 +337,7 @@ class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase): ) -class Fp8PtpcOnlineLinearMethod(_Fp8OnlineLinearBase): +class Fp8PtpcOnlineLinearMethod(OnlineLinearBase): """Online PTPC FP8 linear quantization. Per-output-channel weight scale + dynamic per-token activation scale. The @@ -443,12 +444,12 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase): self, *, weight_block_size: list[int] | None, - layer: torch.nn.Module, + moe: FusedMoEConfig, weight_key: "QuantKey | None" = None, activation_key: "QuantKey | None" = None, allow_vllm_cutlass: bool = False, ): - super().__init__(layer.moe_config) + super().__init__(moe) self.weight_block_size = weight_block_size self.block_quant: bool = self.weight_block_size is not None self.weight_scale_name = ( @@ -555,11 +556,11 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase): def __init__( self, *, - layer: torch.nn.Module, + moe: FusedMoEConfig, ): super().__init__( weight_block_size=None, - layer=layer, + moe=moe, ) def process_weights_after_loading(self, layer: Module) -> None: @@ -612,11 +613,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase): def __init__( self, *, - layer: torch.nn.Module, + moe: FusedMoEConfig, ): super().__init__( weight_block_size=[128, 128], - layer=layer, + moe=moe, ) def maybe_roundup_sizes( @@ -716,13 +717,13 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase): def __init__( self, *, - layer: torch.nn.Module, + moe: FusedMoEConfig, ): from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend super().__init__( weight_block_size=None, - layer=layer, + moe=moe, weight_key=kFp8StaticChannelSym, activation_key=kFp8DynamicTokenSym, allow_vllm_cutlass=True, diff --git a/vllm/model_executor/layers/quantization/online/int8.py b/vllm/model_executor/layers/quantization/online/int8.py index 952229eced..268345ca09 100644 --- a/vllm/model_executor/layers/quantization/online/int8.py +++ b/vllm/model_executor/layers/quantization/online/int8.py @@ -12,6 +12,7 @@ if TYPE_CHECKING: ) from vllm.model_executor.layers.fused_moe import RoutedExperts +from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig from vllm.model_executor.layers.fused_moe.oracle.int8 import ( convert_to_int8_moe_kernel_format, make_int8_moe_kernel, @@ -38,9 +39,9 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase): def __init__( self, *, - layer: torch.nn.Module, + moe: FusedMoEConfig, ): - super().__init__(layer.moe_config) + super().__init__(moe) self.int8_backend, self.experts_cls = select_int8_moe_backend( config=self.moe, weight_key=kInt8StaticChannelSym, diff --git a/vllm/model_executor/layers/quantization/online/mxfp4.py b/vllm/model_executor/layers/quantization/online/mxfp4.py index d09e9ceb6b..0115988e92 100644 --- a/vllm/model_executor/layers/quantization/online/mxfp4.py +++ b/vllm/model_executor/layers/quantization/online/mxfp4.py @@ -17,6 +17,7 @@ if TYPE_CHECKING: ) from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel +from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( TRITON_BACKENDS, Mxfp4MoeBackend, @@ -27,7 +28,7 @@ from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( select_mxfp4_moe_backend, ) from vllm.model_executor.layers.quantization.online.fp8 import ( - _Fp8OnlineLinearBase, + OnlineLinearBase, ) from vllm.model_executor.layers.quantization.online.moe_base import ( OnlineMoEMethodBase, @@ -73,7 +74,7 @@ def _quantize_mxfp4_moe_weight( return w_quant, w_scales -class Mxfp4OnlineLinearMethod(_Fp8OnlineLinearBase): +class Mxfp4OnlineLinearMethod(OnlineLinearBase): """Online MXFP4 linear method. Loads bf16/fp16 checkpoints and quantizes weights to MXFP4 (microscaling FP4 with block-32 scales) during weight loading. @@ -144,8 +145,8 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase): experts_cls: "type[mk.FusedMoEExperts] | None" activation_quant_key = kMxfp4Dynamic - def __init__(self, *, layer: torch.nn.Module): - super().__init__(layer.moe_config) + def __init__(self, *, moe: FusedMoEConfig): + super().__init__(moe) self.weight_block_size: list[int] = [1, MXFP4_BLOCK_SIZE] self.weight_scale_name = "weight_scale" diff --git a/vllm/model_executor/layers/quantization/online/mxfp8.py b/vllm/model_executor/layers/quantization/online/mxfp8.py index 5eb54f9a09..5165762d5b 100644 --- a/vllm/model_executor/layers/quantization/online/mxfp8.py +++ b/vllm/model_executor/layers/quantization/online/mxfp8.py @@ -8,6 +8,8 @@ from typing import TYPE_CHECKING import torch from torch.nn import Module +from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig + if TYPE_CHECKING: import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.model_executor.layers.fused_moe import ( @@ -21,7 +23,7 @@ from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import ( select_mxfp8_moe_backend, ) from vllm.model_executor.layers.quantization.online.fp8 import ( - _Fp8OnlineLinearBase, + OnlineLinearBase, ) from vllm.model_executor.layers.quantization.online.moe_base import ( OnlineMoEMethodBase, @@ -34,7 +36,7 @@ from vllm.model_executor.utils import replace_parameter from vllm.platforms import current_platform -class Mxfp8OnlineLinearMethod(_Fp8OnlineLinearBase): +class Mxfp8OnlineLinearMethod(OnlineLinearBase): """Online MXFP8 linear method. Loads bf16/fp16 checkpoints and quantizes weights to MXFP8 (microscaling FP8 with block-32 scales) during weight loading. @@ -100,8 +102,8 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase): fp8_backend: "Fp8MoeBackend" experts_cls: "type[mk.FusedMoEExperts] | None" - def __init__(self, *, layer: torch.nn.Module): - super().__init__(layer.moe_config) + def __init__(self, *, moe: FusedMoEConfig): + super().__init__(moe) self.weight_block_size: list[int] = [1, MXFP8_BLOCK_SIZE] self.weight_scale_name = "weight_scale" diff --git a/vllm/model_executor/layers/quantization/online/nvfp4.py b/vllm/model_executor/layers/quantization/online/nvfp4.py index de920db901..7abefce257 100644 --- a/vllm/model_executor/layers/quantization/online/nvfp4.py +++ b/vllm/model_executor/layers/quantization/online/nvfp4.py @@ -6,7 +6,10 @@ from torch.nn import Module from vllm._custom_ops import scaled_fp4_quant from vllm.model_executor.layers.fused_moe import RoutedExperts -from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEQuantConfig, +) from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import ( convert_to_nvfp4_moe_kernel_format, make_nvfp4_moe_kernel, @@ -87,13 +90,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase): def __init__( self, *, - layer: torch.nn.Module, + moe: FusedMoEConfig, ): if not current_platform.is_device_capability_family(100): raise ValueError( "nvfp4_per_token online quantization requires a Blackwell (SM100) GPU." ) - super().__init__(layer.moe_config) + super().__init__(moe) self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend( config=self.moe, weight_key=kNvfp4Static, diff --git a/vllm/model_executor/layers/vocab_parallel_embedding.py b/vllm/model_executor/layers/vocab_parallel_embedding.py index 2eddd0ffec..ba901550e8 100644 --- a/vllm/model_executor/layers/vocab_parallel_embedding.py +++ b/vllm/model_executor/layers/vocab_parallel_embedding.py @@ -24,6 +24,7 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase, method_has_implemented_embedding, + resolve_quant_method, ) from vllm.model_executor.layers.utils import dispatch_unquantized_gemm from vllm.model_executor.parameter import BasevLLMParameter @@ -291,7 +292,7 @@ class VocabParallelEmbedding(PluggableLayer): # Avoid overriding a preselected model-specific method with generic # config-based dispatch. if quant_method is None and quant_config is not None: - quant_method = quant_config.get_quant_method(self, prefix=prefix) + quant_method = resolve_quant_method(quant_config, self, prefix=prefix) if quant_method is None: quant_method = UnquantizedEmbeddingMethod() diff --git a/vllm/model_executor/model_loader/base_loader.py b/vllm/model_executor/model_loader/base_loader.py index 3c03be2071..618f874fd0 100644 --- a/vllm/model_executor/model_loader/base_loader.py +++ b/vllm/model_executor/model_loader/base_loader.py @@ -100,6 +100,11 @@ def log_online_quantization(vllm_config: VllmConfig) -> None: ) quant_config = vllm_config.quant_config + online_quantization_config = getattr( + quant_config, "online_quantization_config", None + ) + if isinstance(online_quantization_config, OnlineQuantizationConfig): + quant_config = online_quantization_config if not isinstance(quant_config, OnlineQuantizationConfig): return diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index c8b166d134..93afe87036 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -243,6 +243,26 @@ def get_quant_config( if model_config.quantization is None: raise ValueError("Model quantization method is not specified in the config.") quant_cls = get_quantization_config(model_config.quantization) + from vllm.config.quantization import _ONLINE_SHORTHANDS, QuantizationConfigArgs + from vllm.model_executor.layers.quantization.online.base import ( + OnlineQuantizationConfig, + ) + + # resolve_quantization_config does not resolve `quantization_config` from + # ambiguous shorthands as `"mxfp4"`, `"mxfp8"`. + online_args = model_config.quantization_config + + def maybe_compose_online_quantization( + checkpoint_config: QuantizationConfig, + ) -> QuantizationConfig: + if online_args is None: + return checkpoint_config + assert isinstance(online_args, QuantizationConfigArgs) + + checkpoint_config.online_quantization_config = OnlineQuantizationConfig( + online_args + ) + return checkpoint_config # Read the quantization config from the HF model config, if available. hf_quant_config = getattr(model_config.hf_config, "quantization_config", None) @@ -284,7 +304,9 @@ def get_quant_config( ): pass # fall through to file-based loading below else: - return quant_cls.from_config(hf_quant_config) + return maybe_compose_online_quantization( + quant_cls.from_config(hf_quant_config) + ) # if hf_quant_config is None, we will try to get config from # hf_overrides @@ -302,7 +324,9 @@ def get_quant_config( quantization_config_file = hf_overrides.get("quantization_config_file") if quantization_config_file is not None: if hasattr(quant_cls, "from_config_file"): - return quant_cls.from_config_file(quantization_config_file) + return maybe_compose_online_quantization( + quant_cls.from_config_file(quantization_config_file) + ) else: raise NotImplementedError( "from_config_file is specified in hf_override config, " @@ -312,7 +336,9 @@ def get_quant_config( quantization_config_json = hf_overrides.get("quantization_config_dict_json") if quantization_config_json is not None: if hasattr(quant_cls, "from_config_dict_json"): - return quant_cls.from_config_dict_json(quantization_config_json) + return maybe_compose_online_quantization( + quant_cls.from_config_dict_json(quantization_config_json) + ) else: raise NotImplementedError( "from_config_dict_json is specified in hf_override config, " @@ -320,16 +346,14 @@ def get_quant_config( f"{quant_cls}" ) - # Online quantization doesn't read from checkpoint configs - it quantizes - # fp16/bf16 weights on the fly during loading. - if model_config.quantization_config is not None: - from vllm.config.quantization import QuantizationConfigArgs - from vllm.model_executor.layers.quantization.online.base import ( - OnlineQuantizationConfig, - ) - - assert isinstance(model_config.quantization_config, QuantizationConfigArgs) - return OnlineQuantizationConfig(args=model_config.quantization_config) + # Raw unambiguous online quantization doesn't read from checkpoint configs + # We must continue to load/read the config below in two cases: + # 1. composed online quantization, before `maybe_compose_online_quantization`, + # 2. Ambiguous online shorthands as `"mxpf4"`, `"mxfp8"`, for which `online_args` + # is not set yet. + if quant_cls is OnlineQuantizationConfig and online_args is not None: + assert isinstance(online_args, QuantizationConfigArgs) + return OnlineQuantizationConfig(args=online_args) model_name_or_path = ( maybe_download_from_modelscope( @@ -359,7 +383,11 @@ def get_quant_config( # If the quantization config is not found, use the default config. if not possible_config_filenames: - return quant_cls() + if model_config.quantization in _ONLINE_SHORTHANDS: + args = online_args or _ONLINE_SHORTHANDS[model_config.quantization] + assert isinstance(args, QuantizationConfigArgs) + return OnlineQuantizationConfig(args=args) + return maybe_compose_online_quantization(quant_cls()) config_files = glob.glob(os.path.join(hf_folder, "*.json")) @@ -367,6 +395,10 @@ def get_quant_config( f for f in config_files if any(f.endswith(x) for x in possible_config_filenames) ] if len(quant_config_files) == 0: + if model_config.quantization in _ONLINE_SHORTHANDS: + args = online_args or _ONLINE_SHORTHANDS[model_config.quantization] + assert isinstance(args, QuantizationConfigArgs) + return OnlineQuantizationConfig(args=args) raise ValueError(f"Cannot find the config file for {model_config.quantization}") if len(quant_config_files) > 1: raise ValueError( @@ -380,14 +412,14 @@ def get_quant_config( if model_config.quantization in ("modelopt", "modelopt_mixed"): if config.get("producer", {}).get("name") == "modelopt": - return quant_cls.from_config(config) + return maybe_compose_online_quantization(quant_cls.from_config(config)) else: raise ValueError( f"Unsupported quantization config" f" found for {model_config.quantization} in {f}." ) - return quant_cls.from_config(config) + return maybe_compose_online_quantization(quant_cls.from_config(config)) def get_sparse_attention_config( diff --git a/vllm/models/kimi_k3/nvidia/mla.py b/vllm/models/kimi_k3/nvidia/mla.py index 45ae01606c..9cd261c73a 100644 --- a/vllm/models/kimi_k3/nvidia/mla.py +++ b/vllm/models/kimi_k3/nvidia/mla.py @@ -69,7 +69,10 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, UnquantizedLinearMethod, ) -from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization import ( + QuantizationConfig, + resolve_quant_method, +) from vllm.model_executor.layers.quantization.utils.quant_utils import ( get_and_maybe_dequant_weights, ) @@ -462,7 +465,7 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase): replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True) quant_method = ( - self.quant_config.get_quant_method(self, prefix=self.layer_name) + resolve_quant_method(self.quant_config, self, prefix=self.layer_name) if self.quant_config else None )