[Quantization] Support online quantization with partially pre-quantized checkpoints (#51392)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-09-04 11:46:06 -07:00
committed by GitHub
co-authored by Codex mgoin
parent a11dfcff89
commit 5690b02c03
25 changed files with 846 additions and 175 deletions
+18
View File
@@ -99,6 +99,24 @@ vllm serve openai/gpt-oss-20b --quantization-config.moe.activation mxfp8
Combine with `--moe-backend` to pin a specific kernel family.
### Online quantization on unquantized layers from partially-quantized checkpoints
Online quantization can be used on already quantized checkpoints independently of their original `quant_method` (`modelopt`, `compressed-tensors`, `quark`, etc.), for layers that are left unquantized in the original checkpoint.
The checkpoint `quant_method` remains responsible for its quantized layers, while the selected unquantized layers use the requested online method.
For example:
```bash
vllm serve amd/Qwen3.5-35B-A3B-MXFP4 \
--quantization-config.linear mxfp8
```
adds MXFP8 quantization to the dense linear layers of a Quark checkpoint where only MOE experts are quantized.
!!! info
`quantization_config.ignore` is an online-only exclusion: the original `quant_method` relies solely on its own ignore implementation and on the ignored layers specified in `config.json`.
### Separate Schemes for Dense and MoE Layers
You can apply different quantization schemes to dense linear layers and MoE expert layers via the `linear` and `moe` fields. Each accepts either a full spec dict, or a bare string naming an online shorthand (e.g. `"fp8_per_block"`) or weight format (e.g. `"fp8_per_block_static"`); fields not set fall back to the shorthand defaults.
@@ -83,6 +83,7 @@ def test_fc_layer_quant_config_usage(default_vllm_config, dist_init, device) ->
assert fc_no_quant.output_size == output_size
mock_quant_config = Mock()
mock_quant_config.online_quantization_config = None
fc_with_quant = ReplicatedLinear(
input_size=input_size,
output_size=output_size,
+1 -1
View File
@@ -58,7 +58,7 @@ def test_online_ignore_supports_fnmatch_patterns():
)
with pytest.raises(ValueError, match="matches both quantization_config.ignore"):
config._dispatch_target(layer_name, Mock())
config.resolve_quant_method_cls(Mock(), layer_name)
@pytest.mark.parametrize(
+513 -53
View File
@@ -2,8 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests online quantization."""
import json
from pathlib import Path
from types import SimpleNamespace
from typing import cast
from typing import Any, cast
from unittest.mock import Mock
import pytest
@@ -18,18 +20,39 @@ from tests.quantization.utils import (
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm._custom_ops import scaled_fp4_quant
from vllm.config import ModelConfig
from vllm.config.load import LoadConfig
from vllm.config.model import ModelConfig
from vllm.config.quantization import (
QuantizationConfigArgs,
resolve_quantization_config,
)
from vllm.config.vllm import VllmConfig
from vllm.forward_context import set_forward_context
from vllm.model_executor.kernels.linear.mxfp8.emulation import (
EmulationMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.marlin import (
MarlinMxfp8LinearKernel,
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
LinearBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
CompressedTensorsConfig,
CompressedTensorsLinearMethod,
)
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
CompressedTensorsMoEMethod,
)
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptFp8Config,
ModelOptMxFp8Config,
)
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
@@ -50,19 +73,31 @@ from vllm.model_executor.layers.quantization.online.mxfp4 import (
Mxfp4OnlineLinearMethod,
Mxfp4OnlineMoEMethod,
)
from vllm.model_executor.layers.quantization.online.mxfp8 import (
Mxfp8OnlineLinearMethod,
)
from vllm.model_executor.layers.quantization.online.nvfp4 import (
Nvfp4OnlineMoEMethod,
_quantize_moe_weight_to_nvfp4,
)
from vllm.model_executor.layers.quantization.quark.quark import (
QuarkConfig,
QuarkLinearMethod,
)
from vllm.model_executor.layers.quantization.utils import quant_utils
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_VALUE_DTYPE,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
amax_for_moe_weight_quant,
amax_for_tp_weight_quant,
kMxfp8Dynamic,
weight_amax,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader import weight_utils
from vllm.model_executor.model_loader.base_loader import log_online_quantization
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
from vllm.model_executor.models.granitemoe import (
@@ -83,6 +118,10 @@ else:
DEVICE = current_platform.device_type
GRANITE_MODEL_NAME = "ibm-granite/granite-3.0-1b-a400m-base"
PARTIALLY_PREQUANTIZED_MODEL_NAME = (
"nm-testing/tinysmokeqwen3moe-W4A16-first-only-CTstable"
)
def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
@@ -132,15 +171,349 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
assert process_weights.call_count == 2
def _fully_quantized_quark_config() -> QuarkConfig:
return QuarkConfig(
{
"exclude": [],
"global_quant_config": {
"weight": {
"dtype": "int8",
"qscheme": "per_tensor",
"is_dynamic": False,
"symmetric": True,
},
"input_tensors": {
"dtype": "int8",
"qscheme": "per_tensor",
"is_dynamic": False,
"symmetric": True,
},
},
"layer_quant_config": {},
"layer_type_quant_config": {},
}
)
def _fully_quantized_modelopt_config() -> ModelOptFp8Config:
return ModelOptFp8Config(
quant_method="FP8",
is_checkpoint_fp8_serialized=True,
kv_cache_quant_method=None,
exclude_modules=[],
)
def _moe_only_compressed_tensors_config() -> CompressedTensorsConfig:
return CompressedTensorsConfig(
target_scheme_map={"RoutedExperts": {}},
ignore=[],
quant_format="pack-quantized",
)
def _write_minimal_llama_config(
model_path: Path, quantization_config: dict[str, Any] | None = None
) -> None:
config: dict[str, Any] = {
"architectures": ["LlamaForCausalLM"],
"model_type": "llama",
"hidden_size": 32,
"intermediate_size": 64,
"num_attention_heads": 4,
"num_hidden_layers": 1,
"vocab_size": 32,
"max_position_embeddings": 32,
}
if quantization_config is not None:
config["quantization_config"] = quantization_config
(model_path / "config.json").write_text(json.dumps(config))
@pytest.mark.parametrize(
"checkpoint_config_factory,raises_conflict",
[
pytest.param(_fully_quantized_quark_config, True, id="quark"),
pytest.param(_fully_quantized_modelopt_config, True, id="modelopt"),
pytest.param(
_moe_only_compressed_tensors_config,
False,
id="compressed_tensors",
),
],
)
def test_online_prequantized_compatibility(
checkpoint_config_factory,
raises_conflict: bool,
default_vllm_config,
dist_init,
) -> None:
"""Online weights replace only layers left unquantized by a checkpoint."""
default_vllm_config.model_config = ModelConfig()
checkpoint_config = checkpoint_config_factory()
checkpoint_config.online_quantization_config = OnlineQuantizationConfig(
QuantizationConfigArgs(linear="mxfp8")
)
config = checkpoint_config
layer_kwargs = {
"input_size": 32,
"output_size": 32,
"bias": False,
"params_dtype": torch.bfloat16,
"quant_config": config,
"prefix": "model.layers.0.self_attn.o_proj",
"disable_tp": True,
}
if raises_conflict:
with pytest.raises(ValueError, match="pre-quantized layer"):
ColumnParallelLinear(**layer_kwargs)
else:
layer = ColumnParallelLinear(**layer_kwargs)
assert isinstance(layer.quant_method, Mxfp8OnlineLinearMethod)
def test_online_target_rejects_prequantized_layer(
default_vllm_config, dist_init
) -> None:
"""A targets match participates in checkpoint compatibility checks."""
default_vllm_config.model_config = ModelConfig()
prefix = "model.layers.0.self_attn.o_proj"
quant_config = _fully_quantized_quark_config()
quant_config.online_quantization_config = OnlineQuantizationConfig(
QuantizationConfigArgs(targets={prefix: "mxfp8"})
)
with pytest.raises(ValueError, match="pre-quantized layer"):
ColumnParallelLinear(
input_size=32,
output_size=32,
bias=False,
params_dtype=torch.bfloat16,
quant_config=quant_config,
prefix=prefix,
disable_tp=True,
)
def test_online_ignore_keeps_checkpoint_quantization_linear(
default_vllm_config, dist_init, monkeypatch
):
"""Ignoring online quantization does not replace a checkpoint method (linear)."""
default_vllm_config.model_config = ModelConfig()
quant_config = _fully_quantized_quark_config()
prefix = "model.layers.0.self_attn.o_proj"
quant_config.online_quantization_config = OnlineQuantizationConfig(
QuantizationConfigArgs(linear="mxfp8", ignore=[prefix])
)
monkeypatch.setattr(
quant_config.online_quantization_config,
"get_quant_method",
lambda *args: pytest.fail("online method should not be constructed"),
)
layer = ColumnParallelLinear(
input_size=32,
output_size=32,
bias=False,
params_dtype=torch.bfloat16,
quant_config=quant_config,
prefix=prefix,
disable_tp=True,
)
assert isinstance(layer.quant_method, QuarkLinearMethod)
def test_online_quantization_rejects_prequantized_moe(
default_vllm_config, dist_init
) -> None:
"""Online linear and MoE quantization reject a pre-quantized MoE layer."""
default_vllm_config.model_config = ModelConfig()
prefix = "model.layers.0.mlp.experts"
quant_config = _fully_quantized_quark_config()
quant_config.online_quantization_config = OnlineQuantizationConfig(
QuantizationConfigArgs(linear="mxfp4", moe="mxfp4")
)
with pytest.raises(ValueError, match="pre-quantized layer"):
FusedMoEFactory(
num_experts=4,
top_k=2,
hidden_size=32,
intermediate_size=64,
params_dtype=torch.bfloat16,
quant_config=quant_config,
prefix=prefix,
)
def test_activation_only_override_applies_to_checkpoint_method(
default_vllm_config, tmp_path
) -> None:
"""Activation-only overrides remain available to the checkpoint method."""
_write_minimal_llama_config(tmp_path)
model_config = ModelConfig(
model=str(tmp_path),
quantization="compressed-tensors",
quantization_config=QuantizationConfigArgs(moe={"activation": "mxfp8"}),
hf_overrides={
"quantization_config": {
"quant_method": "compressed-tensors",
"format": "pack-quantized",
"config_groups": {},
}
},
)
result = weight_utils.get_quant_config(model_config, LoadConfig())
assert isinstance(result, CompressedTensorsConfig)
assert isinstance(result.online_quantization_config, OnlineQuantizationConfig)
default_vllm_config.model_config = model_config
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
_resolve_activation_key,
)
assert _resolve_activation_key(None) == kMxfp8Dynamic
def test_online_overlay_requires_normalized_quantization_config(tmp_path) -> None:
"""The loader rejects a raw config dict instead of silently ignoring it."""
_write_minimal_llama_config(tmp_path)
model_config = ModelConfig(
model=str(tmp_path),
quantization="compressed-tensors",
quantization_config={"linear": "mxfp8"},
hf_overrides={
"quantization_config": {
"quant_method": "compressed-tensors",
"format": "pack-quantized",
"config_groups": {},
}
},
)
# assert isinstance(online_args, QuantizationConfigArgs)
# in maybe_compose_online_quantization
with pytest.raises(AssertionError):
weight_utils.get_quant_config(model_config, LoadConfig())
def test_online_overlay_loads_checkpoint_config_file(tmp_path) -> None:
"""An online overlay must not bypass checkpoint config file loading."""
_write_minimal_llama_config(tmp_path)
(tmp_path / "quantize_config.json").write_text(
'{"w_bit": 4, "q_group_size": 128, "zero_point": true}'
)
model_config = ModelConfig(
model=str(tmp_path),
quantization="awq",
quantization_config=QuantizationConfigArgs(linear="mxfp8"),
)
result = weight_utils.get_quant_config(model_config, LoadConfig())
assert isinstance(result, AutoAWQConfig)
assert result.weight_bits == 4
assert result.group_size == 128
assert result.zero_point
assert isinstance(result.online_quantization_config, OnlineQuantizationConfig)
assert result.online_quantization_config.args is model_config.quantization_config
@pytest.mark.parametrize(
"quantization,checkpoint_quantization_config,is_config_file",
[
pytest.param("fp8_per_block", None, False, id="fp8-per-block-online"),
pytest.param("mxfp4", None, False, id="mxfp4-online"),
pytest.param("mxfp8", None, False, id="mxfp8-online"),
pytest.param(
"mxfp8",
{"quant_method": "mxfp8", "ignored_layers": []},
False,
id="mxfp8-inline-checkpoint-config",
),
pytest.param(
"mxfp8",
{"quant_method": "mxfp8", "ignored_layers": []},
True,
id="mxfp8-checkpoint-config-file",
),
],
)
def test_online_shorthand_selects_checkpoint_or_online_config(
tmp_path, quantization, checkpoint_quantization_config, is_config_file
) -> None:
"""Online shorthands defer to checkpoint metadata when available."""
if is_config_file:
(tmp_path / "hf_quant_config.json").write_text(
'{"quant_method": "mxfp8", "ignored_layers": []}'
)
_write_minimal_llama_config(
tmp_path,
None if is_config_file else checkpoint_quantization_config,
)
model_config = ModelConfig(model=str(tmp_path), quantization=quantization)
result = weight_utils.get_quant_config(model_config, LoadConfig())
if checkpoint_quantization_config is not None:
assert isinstance(result, ModelOptMxFp8Config)
assert result.online_quantization_config is None
else:
assert isinstance(result, OnlineQuantizationConfig)
assert result.online_quantization_config is None
def test_log_online_quantization_for_composable_config(monkeypatch) -> None:
"""Composable configs log their nested online quantization results."""
online_config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8"))
online_config.quantized_layers = {
"model.layers.0.self_attn.o_proj": ("linear", "mxfp8", None),
"model.layers.1.self_attn.o_proj": ("linear", "mxfp8", None),
}
vllm_config = VllmConfig(quant_config=online_config)
log_args = []
monkeypatch.setattr(
"vllm.model_executor.model_loader.base_loader.logger.info",
lambda *args: log_args.append(args),
)
log_online_quantization(vllm_config)
assert log_args == [
(
"Quantized %d layers of types: %s",
2,
"; ".join(online_config.quantized_layer_summaries),
)
]
def test_checkpoint_quantization_rejects_online_shorthand(tmp_path) -> None:
"""A checkpoint quant method cannot be replaced by an online shorthand."""
_write_minimal_llama_config(tmp_path, {"quant_method": "mxfp4"})
with pytest.raises(ValueError, match="does not match the quantization"):
ModelConfig(model=str(tmp_path), quantization="fp8_per_channel")
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",
)
@pytest.mark.parametrize(
"quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls",
("model_name,quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls"),
[
# simple case - quantization='fp8_per_tensor'
(
GRANITE_MODEL_NAME,
"fp8_per_tensor",
None,
Fp8PerTensorOnlineLinearMethod,
@@ -148,12 +521,14 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
),
# simple case - quantization='fp8_per_block'
(
GRANITE_MODEL_NAME,
"fp8_per_block",
None,
Fp8PerBlockOnlineLinearMethod,
Fp8PerBlockOnlineMoEMethod,
),
(
GRANITE_MODEL_NAME,
"fp8_per_channel",
None,
Fp8PtpcOnlineLinearMethod,
@@ -161,6 +536,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
),
# quantization='online' with per-layer-kind overrides
(
GRANITE_MODEL_NAME,
"online",
{
"linear": "fp8_per_block",
@@ -171,6 +547,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
),
# quantization='online' with per-layer target patterns
(
GRANITE_MODEL_NAME,
"online",
{
"targets": {
@@ -183,6 +560,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
),
# ignore with direct layer name
(
GRANITE_MODEL_NAME,
"fp8_per_tensor",
# qkv_proj is fused from q_proj/k_proj/v_proj. The shard regex
# remains supported alongside direct fused-name regexes.
@@ -191,11 +569,20 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
Fp8PerTensorOnlineMoEMethod,
),
(
GRANITE_MODEL_NAME,
"mxfp4",
None,
Mxfp4OnlineLinearMethod,
Mxfp4OnlineMoEMethod,
),
pytest.param(
PARTIALLY_PREQUANTIZED_MODEL_NAME,
None,
{"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}},
Mxfp8OnlineLinearMethod,
CompressedTensorsMoEMethod,
id="partially_prequantized_checkpoint",
),
],
ids=[
"fp8_per_tensor",
@@ -205,13 +592,15 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
"targets",
"ignore",
"mxfp4",
"partially_prequantized_checkpoint",
],
)
@pytest.mark.parametrize(
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
)
def test_online_quantization(
quant_scheme: str,
model_name: str,
quant_scheme: str | None,
online_quant_args: dict | None,
expected_linear_cls,
expected_moe_cls,
@@ -243,38 +632,74 @@ def test_online_quantization(
):
pytest.skip(f"Skip test for online {quant_scheme} on XPU platform.")
model_name = "ibm-granite/granite-3.0-1b-a400m-base"
model, vllm_config = load_model_without_vllm_runner(
model_name,
dtype="bfloat16",
quantization=quant_scheme,
model_config_kwargs={
"quantization_config": resolve_quantization_config(
quant_scheme, online_quant_args
),
"hf_overrides": {
"num_hidden_layers": 3,
"vocab_size": 256,
"hidden_size": 256,
"intermediate_size": 512,
"num_attention_heads": 4,
"num_key_value_heads": 4,
"max_position_embeddings": 64,
"num_local_experts": 4,
"num_experts_per_tok": 2,
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
model, vllm_config = load_model_without_vllm_runner(
model_name,
model_config_kwargs={
"quantization_config": resolve_quantization_config(
None, online_quant_args
)
},
},
model_loader_cls=DummyModelLoader,
)
model_loader_cls=DummyModelLoader,
)
assert isinstance(
model.model.layers[1].self_attn.o_proj.quant_method,
expected_linear_cls,
)
assert isinstance(
model.model.layers[0].mlp.experts._quant_method,
expected_moe_cls,
)
assert isinstance(
model.model.layers[0].self_attn.o_proj.quant_method,
CompressedTensorsLinearMethod,
)
else:
model, vllm_config = load_model_without_vllm_runner(
model_name,
dtype="bfloat16",
quantization=quant_scheme,
model_config_kwargs={
"quantization_config": resolve_quantization_config(
quant_scheme, online_quant_args
),
"hf_overrides": {
"num_hidden_layers": 3,
"vocab_size": 256,
"hidden_size": 256,
"intermediate_size": 512,
"num_attention_heads": 4,
"num_key_value_heads": 4,
"max_position_embeddings": 64,
"num_local_experts": 4,
"num_experts_per_tok": 2,
},
},
model_loader_cls=DummyModelLoader,
)
monkeypatch.setattr(Attention, "forward", lambda _, q, k, v: q.contiguous())
o_proj = model.model.layers[0].self_attn.o_proj
moe = model.model.layers[0].block_sparse_moe.experts
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
o_proj = model.model.layers[1].self_attn.o_proj
moe = model.model.layers[0].mlp.experts
else:
o_proj = model.model.layers[0].self_attn.o_proj
moe = model.model.layers[0].block_sparse_moe.experts
assert isinstance(o_proj.quant_method, expected_linear_cls)
assert isinstance(moe._quant_method, expected_moe_cls)
if quant_scheme == "mxfp4":
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance(
o_proj.quant_method.kernel, MarlinMxfp8LinearKernel
):
assert o_proj.weight.dtype == torch.int32
elif model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance(
o_proj.quant_method.kernel, EmulationMxfp8LinearKernel
):
assert o_proj.weight.dtype == torch.bfloat16
elif model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
assert o_proj.weight.dtype == MXFP8_VALUE_DTYPE
elif quant_scheme == "mxfp4":
assert o_proj.weight.dtype == torch.uint8
elif current_platform.is_cuda() or current_platform.is_xpu():
assert o_proj.weight.dtype == torch.float8_e4m3fn
@@ -301,7 +726,11 @@ def test_online_quantization(
layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod
)
if isinstance(online_quant_args, dict) and "targets" in online_quant_args:
if (
model_name == GRANITE_MODEL_NAME
and isinstance(online_quant_args, dict)
and "targets" in online_quant_args
):
for layer in model.model.layers:
assert isinstance(
layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod
@@ -319,8 +748,13 @@ def test_online_quantization(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",
)
def test_online_quantization_loads_real_weights(vllm_runner, monkeypatch) -> None:
"""Verify online quantization loads a Granite-MoE checkpoint end to end."""
@pytest.mark.parametrize(
"model_name", [GRANITE_MODEL_NAME, PARTIALLY_PREQUANTIZED_MODEL_NAME]
)
def test_online_quantization_loads_real_weights(
model_name, vllm_runner, monkeypatch
) -> None:
"""Verify online quantization loads real checkpoints end to end."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
@@ -336,26 +770,52 @@ def test_online_quantization_loads_real_weights(vllm_runner, monkeypatch) -> Non
monkeypatch.setattr(GraniteMoeModel, "load_weights", load_weights)
with vllm_runner(
"ibm-granite/granite-3.0-1b-a400m-base",
quantization="fp8_per_tensor",
dtype="bfloat16",
enforce_eager=True,
hf_overrides={"num_hidden_layers": 3},
max_model_len=16,
max_num_seqs=1,
) as llm:
if model_name == GRANITE_MODEL_NAME:
runner_kwargs = {
"quantization": "fp8_per_tensor",
"dtype": "bfloat16",
"enforce_eager": True,
"hf_overrides": {"num_hidden_layers": 3},
"max_model_len": 16,
"max_num_seqs": 1,
}
else:
runner_kwargs = {
"quantization_config": {
"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}
},
"enforce_eager": True,
}
def check_model(model):
layer = model.model.layers[0]
assert isinstance(
layer.self_attn.o_proj.quant_method,
Fp8PerTensorOnlineLinearMethod,
)
assert isinstance(
layer.block_sparse_moe.experts._quant_method,
Fp8PerTensorOnlineMoEMethod,
)
with vllm_runner(model_name, **runner_kwargs) as llm:
if model_name == GRANITE_MODEL_NAME:
def check_model(model):
layer = model.model.layers[0]
assert isinstance(
layer.self_attn.o_proj.quant_method,
Fp8PerTensorOnlineLinearMethod,
)
assert isinstance(
layer.block_sparse_moe.experts._quant_method,
Fp8PerTensorOnlineMoEMethod,
)
else:
def check_model(model):
assert isinstance(
model.model.layers[1].self_attn.o_proj.quant_method,
Mxfp8OnlineLinearMethod,
)
assert isinstance(
model.model.layers[0].self_attn.o_proj.quant_method,
CompressedTensorsLinearMethod,
)
assert isinstance(
model.model.layers[0].mlp.experts._quant_method,
CompressedTensorsMoEMethod,
)
llm.apply_model(check_model)
outputs = llm.generate_greedy(["Hello my name is"], max_tokens=1)
@@ -457,8 +917,8 @@ def test_online_quantization_targets_ignore_collision() -> None:
)
)
with pytest.raises(ValueError, match="matches both quantization_config.ignore"):
config._dispatch_target(
"model.layers.0.self_attn.o_proj", Mock(spec=LinearBase)
config.resolve_quant_method_cls(
Mock(spec=LinearBase), "model.layers.0.self_attn.o_proj"
)
+1 -1
View File
@@ -354,7 +354,7 @@ def test_online_mxfp4_moe_matches_quark(
weight_quant_key=kMxfp4Static,
activation_quant_key=kMxfp4Dynamic,
)
online_method = Mxfp4OnlineMoEMethod(layer=online_layer)
online_method = Mxfp4OnlineMoEMethod(moe=online_layer.moe_config)
# `RoutedExperts.__init__` applies this round-up in production; these
# layers are built without a quant config, so it is applied explicitly.
@@ -95,6 +95,12 @@ def test_resolve_shorthand_only_populates_both_slots():
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
@pytest.mark.parametrize("quantization", ["mxfp4", "mxfp8"])
def test_resolve_colliding_shorthand_is_deferred(quantization: str):
"""Checkpoint metadata determines whether an MXFP shorthand is online."""
assert resolve_quantization_config(quantization, None) is None
def test_resolve_int8_shorthand_leaves_linear_unset():
# int8_per_channel_weight_only is MoE-only; linear stays None so that
# OnlineQuantizationConfig leaves Linear layers in full precision.
@@ -122,12 +128,9 @@ def test_resolve_merges_explicit_over_shorthand():
assert args.moe == QuantSpec(weight=kFp8StaticTensorSym)
def test_resolve_rejects_quantization_config_with_non_shorthand_quant():
# If --quantization names something other than an online shorthand,
# quantization_config is not allowed via this path (checkpoint quant
# paths read it directly off ModelConfig instead).
with pytest.raises(ValueError, match="quantization_config is only supported"):
resolve_quantization_config("gptq", {"linear": "fp8_per_block"})
def test_resolve_quantization_config_with_checkpoint_quantization():
args = resolve_quantization_config("gptq", {"linear": "fp8_per_block"})
assert args == QuantizationConfigArgs(linear="fp8_per_block")
# ---- QUANT_KEY_NAMES coverage --------------------------------------------
@@ -206,6 +209,6 @@ def test_targets_reject_moe_only_shorthand_for_linear_layer():
)
with pytest.raises(ValueError, match="does not define a QuantSpec"):
config._dispatch_target(
"model.layers.0.self_attn.o_proj", Mock(spec=LinearBase)
config.resolve_quant_method_cls(
Mock(spec=LinearBase), "model.layers.0.self_attn.o_proj"
)
+19 -7
View File
@@ -225,6 +225,10 @@ ONLINE_QUANT_SHORTHAND_NAMES: tuple[str, ...] = (
"online",
)
# These names are also checkpoint quantization methods. Their online configs
# are resolved only when checkpoint quantization metadata is absent.
_DEFERRED_ONLINE_SHORTHANDS = frozenset(("mxfp4", "mxfp8"))
def resolve_quantization_config(
quantization: str | None,
@@ -239,17 +243,25 @@ def resolve_quantization_config(
take precedence over the shorthand.
"""
if quantization is not None and quantization not in ONLINE_QUANT_SHORTHAND_NAMES:
if quantization_config is not None:
raise ValueError(
f"quantization_config is only supported when quantization is "
f"one of {sorted(ONLINE_QUANT_SHORTHAND_NAMES)}, "
f"got quantization={quantization!r}"
)
return None
# Pre-quantized checkpoints can be composed with online quantization
# for layers that the base quant_method leaves unquantized. The
# checkpoint quant_method remains the primary quantization method; composition
# is performed after its config has been loaded.
if quantization_config is None:
return None
# `quantization_config` may hold both:
# 1. Base quantization method activation key override,
# 2. online quantization config to apply on top of the base quant_method.
if isinstance(quantization_config, dict):
return QuantizationConfigArgs(**quantization_config)
return quantization_config
base = _ONLINE_SHORTHANDS.get(quantization) if quantization else None
if quantization_config is None:
if quantization in _DEFERRED_ONLINE_SHORTHANDS:
return None
return base
if isinstance(quantization_config, dict):
@@ -19,7 +19,10 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.linear import (
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization import (
QuantizationConfig,
resolve_quant_method,
)
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
@@ -183,7 +186,9 @@ def _init_kv_cache_quant(
layer._o_scale_float = None
quant_method = (
quant_config.get_quant_method(layer, prefix=prefix) if quant_config else None
resolve_quant_method(quant_config, layer, prefix=prefix)
if quant_config
else None
)
# See [Note: Register q/k/v/prob scales in state dict]
@@ -584,7 +589,7 @@ class Attention(nn.Module, AttentionLayerBase):
# as the default value. See [Note: Register q/k/v/prob scales in state dict]
# for more details.
quant_method = (
self.quant_config.get_quant_method(self, prefix=self.layer_name)
resolve_quant_method(self.quant_config, self, prefix=self.layer_name)
if self.quant_config
else None
)
@@ -246,7 +246,10 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization import (
QuantizationConfig,
resolve_quant_method,
)
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
@@ -1255,7 +1258,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
# as the default value. See [Note: Register q/k/v/prob scales in state dict]
# for more details.
quant_method = (
self.quant_config.get_quant_method(self, prefix=self.layer_name)
resolve_quant_method(self.quant_config, self, prefix=self.layer_name)
if self.quant_config
else None
)
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
)
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
resolve_quant_method,
)
if TYPE_CHECKING:
@@ -199,7 +200,7 @@ class RoutedExperts(PluggableLayer):
"""
quant_method = None
if quant_config is not None:
quant_method = quant_config.get_quant_method(self, prefix)
quant_method = resolve_quant_method(quant_config, self, prefix)
if quant_method is None:
quant_method = UnquantizedFusedMoEMethod(moe_config)
assert isinstance(quant_method, FusedMoEMethodBase)
+2 -1
View File
@@ -24,6 +24,7 @@ from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.determinism.batch_invariant import (
linear_batch_invariant,
)
from vllm.model_executor.layers.quantization import resolve_quant_method
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
@@ -282,7 +283,7 @@ class LinearBase(PluggableLayer):
self.quant_method: QuantizeMethodBase
if quant_config is None:
self.quant_method = UnquantizedLinearMethod()
elif quant_method := quant_config.get_quant_method(self, prefix=prefix):
elif quant_method := resolve_quant_method(quant_config, self, prefix=prefix):
self.quant_method = quant_method
else:
raise ValueError("All linear layers should support quant method.")
@@ -4,7 +4,10 @@
from typing import Literal, get_args
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
resolve_quant_method,
)
from vllm.platforms import current_platform
logger = init_logger(__name__)
@@ -182,6 +185,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
__all__ = [
"QuantizationConfig",
"resolve_quant_method",
"QuantizationMethods",
"get_quantization_config",
"register_quantization_config",
@@ -12,6 +12,9 @@ from transformers import PretrainedConfig
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization import QuantizationMethods
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
from vllm.model_executor.models.utils import WeightsMapper
else:
QuantizationMethods = str
@@ -105,6 +108,7 @@ class QuantizationConfig(ABC):
"""Suffixes of quantization parameters that may be present in the checkpoint but
not in the model, and should be ignored if unexpected during loading. These are used
after remapping, so should be in vLLM format (e.g. .q_scale, not .q.scale)."""
online_quantization_config: "OnlineQuantizationConfig | None" = None
def __init__(self):
super().__init__()
@@ -187,7 +191,8 @@ class QuantizationConfig(ABC):
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> QuantizeMethodBase | None:
"""Get the quantize method to use for the quantized layer.
"""Get the quantize method to use for the quantized layer, from the
pre-quantized checkpoint quant_method.
Args:
layer: The layer for the quant method.
@@ -265,3 +270,46 @@ class QuantizationConfig(ABC):
# TODO: revision is never passed currently in vllm.py,
# but is used in subclasses, should we remove this parameter?
pass
def resolve_quant_method(
quant_config: QuantizationConfig, layer: torch.nn.Module, prefix: str
) -> QuantizeMethodBase | None:
"""Return the checkpoint method with configured online quantization."""
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.linear import (
LinearBase,
UnquantizedLinearMethod,
)
base_quant_method = quant_config.get_quant_method(layer, prefix)
if quant_config.online_quantization_config is None:
return base_quant_method
# Online quantization currently supports only LinearBase and RoutedExperts.
# Embeddings and ParallelLMHead retain their checkpoint quantization method.
if not isinstance(layer, (LinearBase, RoutedExperts)):
return base_quant_method
quant_config.online_quantization_config.packed_modules_mapping = (
quant_config.packed_modules_mapping
)
checkpoint_is_quantized = base_quant_method is not None and not isinstance(
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
)
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
layer, prefix
)
if checkpoint_is_quantized:
if online_target is not None:
raise ValueError(
f"Cannot apply requested online quantization {online_target[3]} to "
f"pre-quantized layer {prefix}: {base_quant_method} was already "
"selected by the checkpoint quantization config."
)
return base_quant_method
if online_target is None:
return base_quant_method
return quant_config.online_quantization_config.get_quant_method(layer, prefix)
@@ -56,5 +56,5 @@ class ExpertsInt8Config(QuantizationConfig):
if isinstance(layer, LinearBase):
return UnquantizedLinearMethod()
elif isinstance(layer, RoutedExperts):
return Int8OnlineMoEMethod(layer=layer)
return Int8OnlineMoEMethod(moe=layer.moe_config)
return None
@@ -219,7 +219,7 @@ class Fp8Config(QuantizationConfig):
Fp8PerTensorOnlineMoEMethod,
)
return Fp8PerTensorOnlineMoEMethod(layer=layer)
return Fp8PerTensorOnlineMoEMethod(moe=layer.moe_config)
elif isinstance(layer, Attention):
return Fp8KVCacheMethod(self)
return None
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
from enum import Enum
from types import MappingProxyType
from typing import Any
@@ -14,6 +15,7 @@ from vllm.config.quantization import (
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoEMethodBase,
RoutedExperts,
)
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
@@ -38,6 +40,7 @@ from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PerTensorOnlineMoEMethod,
Fp8PtpcOnlineLinearMethod,
Fp8PtpcOnlineMoEMethod,
OnlineLinearBase,
)
from vllm.model_executor.layers.quantization.online.int8 import (
Int8OnlineMoEMethod,
@@ -71,6 +74,14 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
logger = init_logger(__name__)
class OnlineQuantizationSource(str, Enum):
"""Supported online quantization configuration sources."""
linear = "linear" # LinearBase
moe = "moe" # RoutedExperts
targets = "targets"
# Online dispatch tables, keyed by the QuantSpec.weight QuantKey. The
# corresponding method class handles the activation choice via its
# `supported_activation_quant` set.
@@ -200,12 +211,23 @@ class OnlineQuantizationConfig(QuantizationConfig):
"quantization='fp8_per_tensor'/'fp8_per_block' instead."
)
def _dispatch(
def _get_method_cls(
self,
spec: QuantSpec | None,
table: dict[QuantKey, type],
layer: torch.nn.Module,
) -> "QuantizeMethodBase | None":
) -> type | None:
"""Resolve the online method class for a layer's quantization spec.
Args:
spec: Quantization specification to resolve.
table: Mapping from weight quantization keys to method classes.
layer: Layer that will use the resolved method.
Returns:
The matching method class, or None when ``spec`` has no weight
quantization.
"""
if spec is None or spec.weight is None:
return None
cls = table.get(spec.weight)
@@ -223,13 +245,75 @@ class OnlineQuantizationConfig(QuantizationConfig):
f"activation override (activation={spec.activation}) is not "
f"yet supported for online {cls.__name__}"
)
if isinstance(layer, RoutedExperts):
return cls(layer=layer)
return cls()
return cls
def _dispatch_target(
def resolve_quant_method_cls(
self, layer: torch.nn.Module, prefix: str
) -> tuple[OnlineQuantizationSource, str, str | None, QuantSpec, type] | None:
"""Resolve quantization metadata and method class without instantiating it.
Args:
layer: Layer for which to resolve online quantization.
prefix: Fully qualified layer name.
Returns:
A tuple of source, quantization key string, target pattern, spec,
and method class. Returns None when online quantization does not
apply to the layer.
"""
quant_spec: QuantSpec | None
if self.args.targets is not None:
resolved_pattern = self._resolve_targets_quant_method_metadata(
prefix, layer
)
if resolved_pattern is None:
return None
source, quant_key_str, target_pattern, quant_spec, table = resolved_pattern
else:
if isinstance(layer, LinearBase):
source = OnlineQuantizationSource.linear
quant_spec = self.args.linear
table = _ONLINE_LINEAR_METHODS
elif isinstance(layer, RoutedExperts):
source = OnlineQuantizationSource.moe
quant_spec = self.args.moe
table = _ONLINE_MOE_METHODS
else:
return None
if should_ignore_layer(
prefix,
ignore=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
use_fnmatch=True,
):
return None
quant_key_str = str(quant_spec)
target_pattern = None
quant_method_cls = self._get_method_cls(quant_spec, table, layer)
if quant_method_cls is None:
return None
assert quant_spec is not None
return source, quant_key_str, target_pattern, quant_spec, quant_method_cls
def _resolve_targets_quant_method_metadata(
self, prefix: str, layer: torch.nn.Module
) -> "QuantizeMethodBase | None":
) -> (
tuple[OnlineQuantizationSource, str, str, QuantSpec, dict[QuantKey, type]]
| None
):
"""Resolve target-pattern quantization metadata and dispatch table.
Args:
prefix: Fully qualified layer name.
layer: Layer matched against configured target patterns.
Returns:
A tuple of source, quantization key string, target pattern, spec,
and dispatch table. Returns None when no pattern applies or the
layer is ignored.
"""
assert self.args.targets is not None
ignored = should_ignore_layer(
prefix,
@@ -254,7 +338,8 @@ class OnlineQuantizationConfig(QuantizationConfig):
f"targets patterns: {matches}. Each layer may match at most "
f"one target."
)
quant_key_str = self.args.targets[matches[0]]
target_pattern = matches[0]
quant_key_str = self.args.targets[target_pattern]
shorthand = _ONLINE_SHORTHANDS[quant_key_str]
if isinstance(layer, LinearBase):
quant_spec = shorthand.linear
@@ -265,63 +350,44 @@ class OnlineQuantizationConfig(QuantizationConfig):
else:
raise ValueError(
f"Layer {prefix} was matched by quantization_config.targets "
f"({matches[0]}), but online quantization is not supported for "
f"({target_pattern}), but online quantization is not supported for "
f"{type(layer).__name__}."
)
if quant_spec is None:
raise ValueError(
f"targets pattern {matches[0]} = {quant_key_str} does "
f"targets pattern {target_pattern} = {quant_key_str} does "
f"not define a QuantSpec for {type(layer).__name__} layers "
f"(matched at {prefix})."
)
method = self._dispatch(quant_spec, table, layer)
if method is not None:
self.quantized_layers[prefix] = ("targets", quant_key_str, matches[0])
return method
return (
OnlineQuantizationSource.targets,
quant_key_str,
target_pattern,
quant_spec,
table,
)
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
# `targets` takes precedence over `moe` and `linear` and is exclusive.
if self.args.targets is not None:
method = self._dispatch_target(prefix, layer)
if method is not None:
return method
if isinstance(layer, LinearBase):
return UnquantizedLinearMethod()
resolved = self.resolve_quant_method_cls(layer, prefix)
if resolved is not None:
source, quant_key_str, target_pattern, _, quant_method_cls = resolved
self.quantized_layers[prefix] = (
source.value,
quant_key_str,
target_pattern,
)
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
return None
assert issubclass(quant_method_cls, FusedMoEMethodBase)
return quant_method_cls(moe=layer.moe_config)
assert issubclass(quant_method_cls, OnlineLinearBase)
return quant_method_cls()
if isinstance(layer, LinearBase):
if should_ignore_layer(
prefix,
ignore=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
use_fnmatch=True,
):
return UnquantizedLinearMethod()
method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer)
if method is not None:
self.quantized_layers[prefix] = (
"linear",
str(self.args.linear),
None,
)
return method
return UnquantizedLinearMethod()
elif isinstance(layer, RoutedExperts):
if should_ignore_layer(
prefix,
ignore=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
use_fnmatch=True,
):
return UnquantizedFusedMoEMethod(layer.moe_config)
method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer)
if method is not None:
self.quantized_layers[prefix] = ("moe", str(self.args.moe), None)
return method
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
return None
@@ -23,6 +23,7 @@ from vllm.model_executor.kernels.linear.scaled_mm import (
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
select_fp8_moe_backend,
)
@@ -111,7 +112,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
return is_row_parallel or (reduces_output_dim and is_column_parallel)
class _Fp8OnlineLinearBase(LinearMethodBase):
class OnlineLinearBase(LinearMethodBase):
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
weights onto meta device and materializes them just-in-time."""
@@ -155,7 +156,7 @@ class _Fp8OnlineLinearBase(LinearMethodBase):
initialize_online_processing(layer)
class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
"""Online tensorwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
@@ -256,7 +257,7 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
return self.fp8_linear.apply_weights(layer, x, bias)
class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
"""Online blockwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-block during loading."""
@@ -336,7 +337,7 @@ class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
)
class Fp8PtpcOnlineLinearMethod(_Fp8OnlineLinearBase):
class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
"""Online PTPC FP8 linear quantization.
Per-output-channel weight scale + dynamic per-token activation scale. The
@@ -443,12 +444,12 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
self,
*,
weight_block_size: list[int] | None,
layer: torch.nn.Module,
moe: FusedMoEConfig,
weight_key: "QuantKey | None" = None,
activation_key: "QuantKey | None" = None,
allow_vllm_cutlass: bool = False,
):
super().__init__(layer.moe_config)
super().__init__(moe)
self.weight_block_size = weight_block_size
self.block_quant: bool = self.weight_block_size is not None
self.weight_scale_name = (
@@ -555,11 +556,11 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
super().__init__(
weight_block_size=None,
layer=layer,
moe=moe,
)
def process_weights_after_loading(self, layer: Module) -> None:
@@ -612,11 +613,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
super().__init__(
weight_block_size=[128, 128],
layer=layer,
moe=moe,
)
def maybe_roundup_sizes(
@@ -716,13 +717,13 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
super().__init__(
weight_block_size=None,
layer=layer,
moe=moe,
weight_key=kFp8StaticChannelSym,
activation_key=kFp8DynamicTokenSym,
allow_vllm_cutlass=True,
@@ -12,6 +12,7 @@ if TYPE_CHECKING:
)
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
convert_to_int8_moe_kernel_format,
make_int8_moe_kernel,
@@ -38,9 +39,9 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
super().__init__(layer.moe_config)
super().__init__(moe)
self.int8_backend, self.experts_cls = select_int8_moe_backend(
config=self.moe,
weight_key=kInt8StaticChannelSym,
@@ -17,6 +17,7 @@ if TYPE_CHECKING:
)
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
@@ -27,7 +28,7 @@ from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
select_mxfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.online.fp8 import (
_Fp8OnlineLinearBase,
OnlineLinearBase,
)
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
@@ -73,7 +74,7 @@ def _quantize_mxfp4_moe_weight(
return w_quant, w_scales
class Mxfp4OnlineLinearMethod(_Fp8OnlineLinearBase):
class Mxfp4OnlineLinearMethod(OnlineLinearBase):
"""Online MXFP4 linear method.
Loads bf16/fp16 checkpoints and quantizes weights to MXFP4 (microscaling
FP4 with block-32 scales) during weight loading.
@@ -144,8 +145,8 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
experts_cls: "type[mk.FusedMoEExperts] | None"
activation_quant_key = kMxfp4Dynamic
def __init__(self, *, layer: torch.nn.Module):
super().__init__(layer.moe_config)
def __init__(self, *, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_block_size: list[int] = [1, MXFP4_BLOCK_SIZE]
self.weight_scale_name = "weight_scale"
@@ -8,6 +8,8 @@ from typing import TYPE_CHECKING
import torch
from torch.nn import Module
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
if TYPE_CHECKING:
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe import (
@@ -21,7 +23,7 @@ from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
select_mxfp8_moe_backend,
)
from vllm.model_executor.layers.quantization.online.fp8 import (
_Fp8OnlineLinearBase,
OnlineLinearBase,
)
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
@@ -34,7 +36,7 @@ from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
class Mxfp8OnlineLinearMethod(_Fp8OnlineLinearBase):
class Mxfp8OnlineLinearMethod(OnlineLinearBase):
"""Online MXFP8 linear method.
Loads bf16/fp16 checkpoints and quantizes weights to MXFP8 (microscaling
FP8 with block-32 scales) during weight loading.
@@ -100,8 +102,8 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
fp8_backend: "Fp8MoeBackend"
experts_cls: "type[mk.FusedMoEExperts] | None"
def __init__(self, *, layer: torch.nn.Module):
super().__init__(layer.moe_config)
def __init__(self, *, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_block_size: list[int] = [1, MXFP8_BLOCK_SIZE]
self.weight_scale_name = "weight_scale"
@@ -6,7 +6,10 @@ from torch.nn import Module
from vllm._custom_ops import scaled_fp4_quant
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
convert_to_nvfp4_moe_kernel_format,
make_nvfp4_moe_kernel,
@@ -87,13 +90,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
if not current_platform.is_device_capability_family(100):
raise ValueError(
"nvfp4_per_token online quantization requires a Blackwell (SM100) GPU."
)
super().__init__(layer.moe_config)
super().__init__(moe)
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
config=self.moe,
weight_key=kNvfp4Static,
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
method_has_implemented_embedding,
resolve_quant_method,
)
from vllm.model_executor.layers.utils import dispatch_unquantized_gemm
from vllm.model_executor.parameter import BasevLLMParameter
@@ -291,7 +292,7 @@ class VocabParallelEmbedding(PluggableLayer):
# Avoid overriding a preselected model-specific method with generic
# config-based dispatch.
if quant_method is None and quant_config is not None:
quant_method = quant_config.get_quant_method(self, prefix=prefix)
quant_method = resolve_quant_method(quant_config, self, prefix=prefix)
if quant_method is None:
quant_method = UnquantizedEmbeddingMethod()
@@ -100,6 +100,11 @@ def log_online_quantization(vllm_config: VllmConfig) -> None:
)
quant_config = vllm_config.quant_config
online_quantization_config = getattr(
quant_config, "online_quantization_config", None
)
if isinstance(online_quantization_config, OnlineQuantizationConfig):
quant_config = online_quantization_config
if not isinstance(quant_config, OnlineQuantizationConfig):
return
@@ -243,6 +243,26 @@ def get_quant_config(
if model_config.quantization is None:
raise ValueError("Model quantization method is not specified in the config.")
quant_cls = get_quantization_config(model_config.quantization)
from vllm.config.quantization import _ONLINE_SHORTHANDS, QuantizationConfigArgs
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
# resolve_quantization_config does not resolve `quantization_config` from
# ambiguous shorthands as `"mxfp4"`, `"mxfp8"`.
online_args = model_config.quantization_config
def maybe_compose_online_quantization(
checkpoint_config: QuantizationConfig,
) -> QuantizationConfig:
if online_args is None:
return checkpoint_config
assert isinstance(online_args, QuantizationConfigArgs)
checkpoint_config.online_quantization_config = OnlineQuantizationConfig(
online_args
)
return checkpoint_config
# Read the quantization config from the HF model config, if available.
hf_quant_config = getattr(model_config.hf_config, "quantization_config", None)
@@ -284,7 +304,9 @@ def get_quant_config(
):
pass # fall through to file-based loading below
else:
return quant_cls.from_config(hf_quant_config)
return maybe_compose_online_quantization(
quant_cls.from_config(hf_quant_config)
)
# if hf_quant_config is None, we will try to get config from
# hf_overrides
@@ -302,7 +324,9 @@ def get_quant_config(
quantization_config_file = hf_overrides.get("quantization_config_file")
if quantization_config_file is not None:
if hasattr(quant_cls, "from_config_file"):
return quant_cls.from_config_file(quantization_config_file)
return maybe_compose_online_quantization(
quant_cls.from_config_file(quantization_config_file)
)
else:
raise NotImplementedError(
"from_config_file is specified in hf_override config, "
@@ -312,7 +336,9 @@ def get_quant_config(
quantization_config_json = hf_overrides.get("quantization_config_dict_json")
if quantization_config_json is not None:
if hasattr(quant_cls, "from_config_dict_json"):
return quant_cls.from_config_dict_json(quantization_config_json)
return maybe_compose_online_quantization(
quant_cls.from_config_dict_json(quantization_config_json)
)
else:
raise NotImplementedError(
"from_config_dict_json is specified in hf_override config, "
@@ -320,16 +346,14 @@ def get_quant_config(
f"{quant_cls}"
)
# Online quantization doesn't read from checkpoint configs - it quantizes
# fp16/bf16 weights on the fly during loading.
if model_config.quantization_config is not None:
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
assert isinstance(model_config.quantization_config, QuantizationConfigArgs)
return OnlineQuantizationConfig(args=model_config.quantization_config)
# Raw unambiguous online quantization doesn't read from checkpoint configs
# We must continue to load/read the config below in two cases:
# 1. composed online quantization, before `maybe_compose_online_quantization`,
# 2. Ambiguous online shorthands as `"mxpf4"`, `"mxfp8"`, for which `online_args`
# is not set yet.
if quant_cls is OnlineQuantizationConfig and online_args is not None:
assert isinstance(online_args, QuantizationConfigArgs)
return OnlineQuantizationConfig(args=online_args)
model_name_or_path = (
maybe_download_from_modelscope(
@@ -359,7 +383,11 @@ def get_quant_config(
# If the quantization config is not found, use the default config.
if not possible_config_filenames:
return quant_cls()
if model_config.quantization in _ONLINE_SHORTHANDS:
args = online_args or _ONLINE_SHORTHANDS[model_config.quantization]
assert isinstance(args, QuantizationConfigArgs)
return OnlineQuantizationConfig(args=args)
return maybe_compose_online_quantization(quant_cls())
config_files = glob.glob(os.path.join(hf_folder, "*.json"))
@@ -367,6 +395,10 @@ def get_quant_config(
f for f in config_files if any(f.endswith(x) for x in possible_config_filenames)
]
if len(quant_config_files) == 0:
if model_config.quantization in _ONLINE_SHORTHANDS:
args = online_args or _ONLINE_SHORTHANDS[model_config.quantization]
assert isinstance(args, QuantizationConfigArgs)
return OnlineQuantizationConfig(args=args)
raise ValueError(f"Cannot find the config file for {model_config.quantization}")
if len(quant_config_files) > 1:
raise ValueError(
@@ -380,14 +412,14 @@ def get_quant_config(
if model_config.quantization in ("modelopt", "modelopt_mixed"):
if config.get("producer", {}).get("name") == "modelopt":
return quant_cls.from_config(config)
return maybe_compose_online_quantization(quant_cls.from_config(config))
else:
raise ValueError(
f"Unsupported quantization config"
f" found for {model_config.quantization} in {f}."
)
return quant_cls.from_config(config)
return maybe_compose_online_quantization(quant_cls.from_config(config))
def get_sparse_attention_config(
+5 -2
View File
@@ -69,7 +69,10 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization import (
QuantizationConfig,
resolve_quant_method,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_and_maybe_dequant_weights,
)
@@ -462,7 +465,7 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
quant_method = (
self.quant_config.get_quant_method(self, prefix=self.layer_name)
resolve_quant_method(self.quant_config, self, prefix=self.layer_name)
if self.quant_config
else None
)