[Quantization] Support online quantization with partially pre-quantized checkpoints (#51392)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
a11dfcff89
commit
5690b02c03
@@ -99,6 +99,24 @@ vllm serve openai/gpt-oss-20b --quantization-config.moe.activation mxfp8
|
||||
|
||||
Combine with `--moe-backend` to pin a specific kernel family.
|
||||
|
||||
### Online quantization on unquantized layers from partially-quantized checkpoints
|
||||
|
||||
Online quantization can be used on already quantized checkpoints independently of their original `quant_method` (`modelopt`, `compressed-tensors`, `quark`, etc.), for layers that are left unquantized in the original checkpoint.
|
||||
|
||||
The checkpoint `quant_method` remains responsible for its quantized layers, while the selected unquantized layers use the requested online method.
|
||||
|
||||
For example:
|
||||
|
||||
```bash
|
||||
vllm serve amd/Qwen3.5-35B-A3B-MXFP4 \
|
||||
--quantization-config.linear mxfp8
|
||||
```
|
||||
|
||||
adds MXFP8 quantization to the dense linear layers of a Quark checkpoint where only MOE experts are quantized.
|
||||
|
||||
!!! info
|
||||
`quantization_config.ignore` is an online-only exclusion: the original `quant_method` relies solely on its own ignore implementation and on the ignored layers specified in `config.json`.
|
||||
|
||||
### Separate Schemes for Dense and MoE Layers
|
||||
|
||||
You can apply different quantization schemes to dense linear layers and MoE expert layers via the `linear` and `moe` fields. Each accepts either a full spec dict, or a bare string naming an online shorthand (e.g. `"fp8_per_block"`) or weight format (e.g. `"fp8_per_block_static"`); fields not set fall back to the shorthand defaults.
|
||||
|
||||
@@ -83,6 +83,7 @@ def test_fc_layer_quant_config_usage(default_vllm_config, dist_init, device) ->
|
||||
assert fc_no_quant.output_size == output_size
|
||||
|
||||
mock_quant_config = Mock()
|
||||
mock_quant_config.online_quantization_config = None
|
||||
fc_with_quant = ReplicatedLinear(
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
|
||||
@@ -58,7 +58,7 @@ def test_online_ignore_supports_fnmatch_patterns():
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="matches both quantization_config.ignore"):
|
||||
config._dispatch_target(layer_name, Mock())
|
||||
config.resolve_quant_method_cls(Mock(), layer_name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests online quantization."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
from typing import Any, cast
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
@@ -18,18 +20,39 @@ from tests.quantization.utils import (
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm._custom_ops import scaled_fp4_quant
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config.quantization import (
|
||||
QuantizationConfigArgs,
|
||||
resolve_quantization_config,
|
||||
)
|
||||
from vllm.config.vllm import VllmConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.kernels.linear.mxfp8.emulation import (
|
||||
EmulationMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.marlin import (
|
||||
MarlinMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
||||
CompressedTensorsConfig,
|
||||
CompressedTensorsLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
|
||||
CompressedTensorsMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptFp8Config,
|
||||
ModelOptMxFp8Config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
@@ -50,19 +73,31 @@ from vllm.model_executor.layers.quantization.online.mxfp4 import (
|
||||
Mxfp4OnlineLinearMethod,
|
||||
Mxfp4OnlineMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.mxfp8 import (
|
||||
Mxfp8OnlineLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.nvfp4 import (
|
||||
Nvfp4OnlineMoEMethod,
|
||||
_quantize_moe_weight_to_nvfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.quark import (
|
||||
QuarkConfig,
|
||||
QuarkLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils import quant_utils
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_VALUE_DTYPE,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
amax_for_moe_weight_quant,
|
||||
amax_for_tp_weight_quant,
|
||||
kMxfp8Dynamic,
|
||||
weight_amax,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader import weight_utils
|
||||
from vllm.model_executor.model_loader.base_loader import log_online_quantization
|
||||
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
|
||||
from vllm.model_executor.models.granitemoe import (
|
||||
@@ -83,6 +118,10 @@ else:
|
||||
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
GRANITE_MODEL_NAME = "ibm-granite/granite-3.0-1b-a400m-base"
|
||||
PARTIALLY_PREQUANTIZED_MODEL_NAME = (
|
||||
"nm-testing/tinysmokeqwen3moe-W4A16-first-only-CTstable"
|
||||
)
|
||||
|
||||
|
||||
def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
@@ -132,15 +171,349 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
assert process_weights.call_count == 2
|
||||
|
||||
|
||||
def _fully_quantized_quark_config() -> QuarkConfig:
|
||||
return QuarkConfig(
|
||||
{
|
||||
"exclude": [],
|
||||
"global_quant_config": {
|
||||
"weight": {
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
"input_tensors": {
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
},
|
||||
"layer_quant_config": {},
|
||||
"layer_type_quant_config": {},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _fully_quantized_modelopt_config() -> ModelOptFp8Config:
|
||||
return ModelOptFp8Config(
|
||||
quant_method="FP8",
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
kv_cache_quant_method=None,
|
||||
exclude_modules=[],
|
||||
)
|
||||
|
||||
|
||||
def _moe_only_compressed_tensors_config() -> CompressedTensorsConfig:
|
||||
return CompressedTensorsConfig(
|
||||
target_scheme_map={"RoutedExperts": {}},
|
||||
ignore=[],
|
||||
quant_format="pack-quantized",
|
||||
)
|
||||
|
||||
|
||||
def _write_minimal_llama_config(
|
||||
model_path: Path, quantization_config: dict[str, Any] | None = None
|
||||
) -> None:
|
||||
config: dict[str, Any] = {
|
||||
"architectures": ["LlamaForCausalLM"],
|
||||
"model_type": "llama",
|
||||
"hidden_size": 32,
|
||||
"intermediate_size": 64,
|
||||
"num_attention_heads": 4,
|
||||
"num_hidden_layers": 1,
|
||||
"vocab_size": 32,
|
||||
"max_position_embeddings": 32,
|
||||
}
|
||||
if quantization_config is not None:
|
||||
config["quantization_config"] = quantization_config
|
||||
(model_path / "config.json").write_text(json.dumps(config))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"checkpoint_config_factory,raises_conflict",
|
||||
[
|
||||
pytest.param(_fully_quantized_quark_config, True, id="quark"),
|
||||
pytest.param(_fully_quantized_modelopt_config, True, id="modelopt"),
|
||||
pytest.param(
|
||||
_moe_only_compressed_tensors_config,
|
||||
False,
|
||||
id="compressed_tensors",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_online_prequantized_compatibility(
|
||||
checkpoint_config_factory,
|
||||
raises_conflict: bool,
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
) -> None:
|
||||
"""Online weights replace only layers left unquantized by a checkpoint."""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
checkpoint_config = checkpoint_config_factory()
|
||||
|
||||
checkpoint_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(linear="mxfp8")
|
||||
)
|
||||
config = checkpoint_config
|
||||
|
||||
layer_kwargs = {
|
||||
"input_size": 32,
|
||||
"output_size": 32,
|
||||
"bias": False,
|
||||
"params_dtype": torch.bfloat16,
|
||||
"quant_config": config,
|
||||
"prefix": "model.layers.0.self_attn.o_proj",
|
||||
"disable_tp": True,
|
||||
}
|
||||
|
||||
if raises_conflict:
|
||||
with pytest.raises(ValueError, match="pre-quantized layer"):
|
||||
ColumnParallelLinear(**layer_kwargs)
|
||||
else:
|
||||
layer = ColumnParallelLinear(**layer_kwargs)
|
||||
assert isinstance(layer.quant_method, Mxfp8OnlineLinearMethod)
|
||||
|
||||
|
||||
def test_online_target_rejects_prequantized_layer(
|
||||
default_vllm_config, dist_init
|
||||
) -> None:
|
||||
"""A targets match participates in checkpoint compatibility checks."""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
prefix = "model.layers.0.self_attn.o_proj"
|
||||
quant_config = _fully_quantized_quark_config()
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(targets={prefix: "mxfp8"})
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="pre-quantized layer"):
|
||||
ColumnParallelLinear(
|
||||
input_size=32,
|
||||
output_size=32,
|
||||
bias=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
disable_tp=True,
|
||||
)
|
||||
|
||||
|
||||
def test_online_ignore_keeps_checkpoint_quantization_linear(
|
||||
default_vllm_config, dist_init, monkeypatch
|
||||
):
|
||||
"""Ignoring online quantization does not replace a checkpoint method (linear)."""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
quant_config = _fully_quantized_quark_config()
|
||||
prefix = "model.layers.0.self_attn.o_proj"
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(linear="mxfp8", ignore=[prefix])
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
quant_config.online_quantization_config,
|
||||
"get_quant_method",
|
||||
lambda *args: pytest.fail("online method should not be constructed"),
|
||||
)
|
||||
|
||||
layer = ColumnParallelLinear(
|
||||
input_size=32,
|
||||
output_size=32,
|
||||
bias=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
disable_tp=True,
|
||||
)
|
||||
|
||||
assert isinstance(layer.quant_method, QuarkLinearMethod)
|
||||
|
||||
|
||||
def test_online_quantization_rejects_prequantized_moe(
|
||||
default_vllm_config, dist_init
|
||||
) -> None:
|
||||
"""Online linear and MoE quantization reject a pre-quantized MoE layer."""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
prefix = "model.layers.0.mlp.experts"
|
||||
quant_config = _fully_quantized_quark_config()
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(linear="mxfp4", moe="mxfp4")
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="pre-quantized layer"):
|
||||
FusedMoEFactory(
|
||||
num_experts=4,
|
||||
top_k=2,
|
||||
hidden_size=32,
|
||||
intermediate_size=64,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
|
||||
def test_activation_only_override_applies_to_checkpoint_method(
|
||||
default_vllm_config, tmp_path
|
||||
) -> None:
|
||||
"""Activation-only overrides remain available to the checkpoint method."""
|
||||
_write_minimal_llama_config(tmp_path)
|
||||
model_config = ModelConfig(
|
||||
model=str(tmp_path),
|
||||
quantization="compressed-tensors",
|
||||
quantization_config=QuantizationConfigArgs(moe={"activation": "mxfp8"}),
|
||||
hf_overrides={
|
||||
"quantization_config": {
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": "pack-quantized",
|
||||
"config_groups": {},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
result = weight_utils.get_quant_config(model_config, LoadConfig())
|
||||
|
||||
assert isinstance(result, CompressedTensorsConfig)
|
||||
assert isinstance(result.online_quantization_config, OnlineQuantizationConfig)
|
||||
|
||||
default_vllm_config.model_config = model_config
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
_resolve_activation_key,
|
||||
)
|
||||
|
||||
assert _resolve_activation_key(None) == kMxfp8Dynamic
|
||||
|
||||
|
||||
def test_online_overlay_requires_normalized_quantization_config(tmp_path) -> None:
|
||||
"""The loader rejects a raw config dict instead of silently ignoring it."""
|
||||
_write_minimal_llama_config(tmp_path)
|
||||
model_config = ModelConfig(
|
||||
model=str(tmp_path),
|
||||
quantization="compressed-tensors",
|
||||
quantization_config={"linear": "mxfp8"},
|
||||
hf_overrides={
|
||||
"quantization_config": {
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": "pack-quantized",
|
||||
"config_groups": {},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
# assert isinstance(online_args, QuantizationConfigArgs)
|
||||
# in maybe_compose_online_quantization
|
||||
with pytest.raises(AssertionError):
|
||||
weight_utils.get_quant_config(model_config, LoadConfig())
|
||||
|
||||
|
||||
def test_online_overlay_loads_checkpoint_config_file(tmp_path) -> None:
|
||||
"""An online overlay must not bypass checkpoint config file loading."""
|
||||
_write_minimal_llama_config(tmp_path)
|
||||
(tmp_path / "quantize_config.json").write_text(
|
||||
'{"w_bit": 4, "q_group_size": 128, "zero_point": true}'
|
||||
)
|
||||
model_config = ModelConfig(
|
||||
model=str(tmp_path),
|
||||
quantization="awq",
|
||||
quantization_config=QuantizationConfigArgs(linear="mxfp8"),
|
||||
)
|
||||
|
||||
result = weight_utils.get_quant_config(model_config, LoadConfig())
|
||||
|
||||
assert isinstance(result, AutoAWQConfig)
|
||||
assert result.weight_bits == 4
|
||||
assert result.group_size == 128
|
||||
assert result.zero_point
|
||||
assert isinstance(result.online_quantization_config, OnlineQuantizationConfig)
|
||||
assert result.online_quantization_config.args is model_config.quantization_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"quantization,checkpoint_quantization_config,is_config_file",
|
||||
[
|
||||
pytest.param("fp8_per_block", None, False, id="fp8-per-block-online"),
|
||||
pytest.param("mxfp4", None, False, id="mxfp4-online"),
|
||||
pytest.param("mxfp8", None, False, id="mxfp8-online"),
|
||||
pytest.param(
|
||||
"mxfp8",
|
||||
{"quant_method": "mxfp8", "ignored_layers": []},
|
||||
False,
|
||||
id="mxfp8-inline-checkpoint-config",
|
||||
),
|
||||
pytest.param(
|
||||
"mxfp8",
|
||||
{"quant_method": "mxfp8", "ignored_layers": []},
|
||||
True,
|
||||
id="mxfp8-checkpoint-config-file",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_online_shorthand_selects_checkpoint_or_online_config(
|
||||
tmp_path, quantization, checkpoint_quantization_config, is_config_file
|
||||
) -> None:
|
||||
"""Online shorthands defer to checkpoint metadata when available."""
|
||||
|
||||
if is_config_file:
|
||||
(tmp_path / "hf_quant_config.json").write_text(
|
||||
'{"quant_method": "mxfp8", "ignored_layers": []}'
|
||||
)
|
||||
|
||||
_write_minimal_llama_config(
|
||||
tmp_path,
|
||||
None if is_config_file else checkpoint_quantization_config,
|
||||
)
|
||||
model_config = ModelConfig(model=str(tmp_path), quantization=quantization)
|
||||
|
||||
result = weight_utils.get_quant_config(model_config, LoadConfig())
|
||||
|
||||
if checkpoint_quantization_config is not None:
|
||||
assert isinstance(result, ModelOptMxFp8Config)
|
||||
assert result.online_quantization_config is None
|
||||
else:
|
||||
assert isinstance(result, OnlineQuantizationConfig)
|
||||
assert result.online_quantization_config is None
|
||||
|
||||
|
||||
def test_log_online_quantization_for_composable_config(monkeypatch) -> None:
|
||||
"""Composable configs log their nested online quantization results."""
|
||||
online_config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8"))
|
||||
online_config.quantized_layers = {
|
||||
"model.layers.0.self_attn.o_proj": ("linear", "mxfp8", None),
|
||||
"model.layers.1.self_attn.o_proj": ("linear", "mxfp8", None),
|
||||
}
|
||||
vllm_config = VllmConfig(quant_config=online_config)
|
||||
log_args = []
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.base_loader.logger.info",
|
||||
lambda *args: log_args.append(args),
|
||||
)
|
||||
|
||||
log_online_quantization(vllm_config)
|
||||
|
||||
assert log_args == [
|
||||
(
|
||||
"Quantized %d layers of types: %s",
|
||||
2,
|
||||
"; ".join(online_config.quantized_layer_summaries),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_checkpoint_quantization_rejects_online_shorthand(tmp_path) -> None:
|
||||
"""A checkpoint quant method cannot be replaced by an online shorthand."""
|
||||
_write_minimal_llama_config(tmp_path, {"quant_method": "mxfp4"})
|
||||
|
||||
with pytest.raises(ValueError, match="does not match the quantization"):
|
||||
ModelConfig(model=str(tmp_path), quantization="fp8_per_channel")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls",
|
||||
("model_name,quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls"),
|
||||
[
|
||||
# simple case - quantization='fp8_per_tensor'
|
||||
(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_tensor",
|
||||
None,
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
@@ -148,12 +521,14 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
),
|
||||
# simple case - quantization='fp8_per_block'
|
||||
(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_block",
|
||||
None,
|
||||
Fp8PerBlockOnlineLinearMethod,
|
||||
Fp8PerBlockOnlineMoEMethod,
|
||||
),
|
||||
(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_channel",
|
||||
None,
|
||||
Fp8PtpcOnlineLinearMethod,
|
||||
@@ -161,6 +536,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
),
|
||||
# quantization='online' with per-layer-kind overrides
|
||||
(
|
||||
GRANITE_MODEL_NAME,
|
||||
"online",
|
||||
{
|
||||
"linear": "fp8_per_block",
|
||||
@@ -171,6 +547,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
),
|
||||
# quantization='online' with per-layer target patterns
|
||||
(
|
||||
GRANITE_MODEL_NAME,
|
||||
"online",
|
||||
{
|
||||
"targets": {
|
||||
@@ -183,6 +560,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
),
|
||||
# ignore with direct layer name
|
||||
(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_tensor",
|
||||
# qkv_proj is fused from q_proj/k_proj/v_proj. The shard regex
|
||||
# remains supported alongside direct fused-name regexes.
|
||||
@@ -191,11 +569,20 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
),
|
||||
(
|
||||
GRANITE_MODEL_NAME,
|
||||
"mxfp4",
|
||||
None,
|
||||
Mxfp4OnlineLinearMethod,
|
||||
Mxfp4OnlineMoEMethod,
|
||||
),
|
||||
pytest.param(
|
||||
PARTIALLY_PREQUANTIZED_MODEL_NAME,
|
||||
None,
|
||||
{"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}},
|
||||
Mxfp8OnlineLinearMethod,
|
||||
CompressedTensorsMoEMethod,
|
||||
id="partially_prequantized_checkpoint",
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"fp8_per_tensor",
|
||||
@@ -205,13 +592,15 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
"targets",
|
||||
"ignore",
|
||||
"mxfp4",
|
||||
"partially_prequantized_checkpoint",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
|
||||
)
|
||||
def test_online_quantization(
|
||||
quant_scheme: str,
|
||||
model_name: str,
|
||||
quant_scheme: str | None,
|
||||
online_quant_args: dict | None,
|
||||
expected_linear_cls,
|
||||
expected_moe_cls,
|
||||
@@ -243,38 +632,74 @@ def test_online_quantization(
|
||||
):
|
||||
pytest.skip(f"Skip test for online {quant_scheme} on XPU platform.")
|
||||
|
||||
model_name = "ibm-granite/granite-3.0-1b-a400m-base"
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
model_name,
|
||||
dtype="bfloat16",
|
||||
quantization=quant_scheme,
|
||||
model_config_kwargs={
|
||||
"quantization_config": resolve_quantization_config(
|
||||
quant_scheme, online_quant_args
|
||||
),
|
||||
"hf_overrides": {
|
||||
"num_hidden_layers": 3,
|
||||
"vocab_size": 256,
|
||||
"hidden_size": 256,
|
||||
"intermediate_size": 512,
|
||||
"num_attention_heads": 4,
|
||||
"num_key_value_heads": 4,
|
||||
"max_position_embeddings": 64,
|
||||
"num_local_experts": 4,
|
||||
"num_experts_per_tok": 2,
|
||||
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
model_name,
|
||||
model_config_kwargs={
|
||||
"quantization_config": resolve_quantization_config(
|
||||
None, online_quant_args
|
||||
)
|
||||
},
|
||||
},
|
||||
model_loader_cls=DummyModelLoader,
|
||||
)
|
||||
model_loader_cls=DummyModelLoader,
|
||||
)
|
||||
assert isinstance(
|
||||
model.model.layers[1].self_attn.o_proj.quant_method,
|
||||
expected_linear_cls,
|
||||
)
|
||||
assert isinstance(
|
||||
model.model.layers[0].mlp.experts._quant_method,
|
||||
expected_moe_cls,
|
||||
)
|
||||
assert isinstance(
|
||||
model.model.layers[0].self_attn.o_proj.quant_method,
|
||||
CompressedTensorsLinearMethod,
|
||||
)
|
||||
else:
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
model_name,
|
||||
dtype="bfloat16",
|
||||
quantization=quant_scheme,
|
||||
model_config_kwargs={
|
||||
"quantization_config": resolve_quantization_config(
|
||||
quant_scheme, online_quant_args
|
||||
),
|
||||
"hf_overrides": {
|
||||
"num_hidden_layers": 3,
|
||||
"vocab_size": 256,
|
||||
"hidden_size": 256,
|
||||
"intermediate_size": 512,
|
||||
"num_attention_heads": 4,
|
||||
"num_key_value_heads": 4,
|
||||
"max_position_embeddings": 64,
|
||||
"num_local_experts": 4,
|
||||
"num_experts_per_tok": 2,
|
||||
},
|
||||
},
|
||||
model_loader_cls=DummyModelLoader,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(Attention, "forward", lambda _, q, k, v: q.contiguous())
|
||||
|
||||
o_proj = model.model.layers[0].self_attn.o_proj
|
||||
moe = model.model.layers[0].block_sparse_moe.experts
|
||||
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
|
||||
o_proj = model.model.layers[1].self_attn.o_proj
|
||||
moe = model.model.layers[0].mlp.experts
|
||||
else:
|
||||
o_proj = model.model.layers[0].self_attn.o_proj
|
||||
moe = model.model.layers[0].block_sparse_moe.experts
|
||||
assert isinstance(o_proj.quant_method, expected_linear_cls)
|
||||
assert isinstance(moe._quant_method, expected_moe_cls)
|
||||
|
||||
if quant_scheme == "mxfp4":
|
||||
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance(
|
||||
o_proj.quant_method.kernel, MarlinMxfp8LinearKernel
|
||||
):
|
||||
assert o_proj.weight.dtype == torch.int32
|
||||
elif model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance(
|
||||
o_proj.quant_method.kernel, EmulationMxfp8LinearKernel
|
||||
):
|
||||
assert o_proj.weight.dtype == torch.bfloat16
|
||||
elif model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
|
||||
assert o_proj.weight.dtype == MXFP8_VALUE_DTYPE
|
||||
elif quant_scheme == "mxfp4":
|
||||
assert o_proj.weight.dtype == torch.uint8
|
||||
elif current_platform.is_cuda() or current_platform.is_xpu():
|
||||
assert o_proj.weight.dtype == torch.float8_e4m3fn
|
||||
@@ -301,7 +726,11 @@ def test_online_quantization(
|
||||
layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod
|
||||
)
|
||||
|
||||
if isinstance(online_quant_args, dict) and "targets" in online_quant_args:
|
||||
if (
|
||||
model_name == GRANITE_MODEL_NAME
|
||||
and isinstance(online_quant_args, dict)
|
||||
and "targets" in online_quant_args
|
||||
):
|
||||
for layer in model.model.layers:
|
||||
assert isinstance(
|
||||
layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod
|
||||
@@ -319,8 +748,13 @@ def test_online_quantization(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_online_quantization_loads_real_weights(vllm_runner, monkeypatch) -> None:
|
||||
"""Verify online quantization loads a Granite-MoE checkpoint end to end."""
|
||||
@pytest.mark.parametrize(
|
||||
"model_name", [GRANITE_MODEL_NAME, PARTIALLY_PREQUANTIZED_MODEL_NAME]
|
||||
)
|
||||
def test_online_quantization_loads_real_weights(
|
||||
model_name, vllm_runner, monkeypatch
|
||||
) -> None:
|
||||
"""Verify online quantization loads real checkpoints end to end."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
@@ -336,26 +770,52 @@ def test_online_quantization_loads_real_weights(vllm_runner, monkeypatch) -> Non
|
||||
|
||||
monkeypatch.setattr(GraniteMoeModel, "load_weights", load_weights)
|
||||
|
||||
with vllm_runner(
|
||||
"ibm-granite/granite-3.0-1b-a400m-base",
|
||||
quantization="fp8_per_tensor",
|
||||
dtype="bfloat16",
|
||||
enforce_eager=True,
|
||||
hf_overrides={"num_hidden_layers": 3},
|
||||
max_model_len=16,
|
||||
max_num_seqs=1,
|
||||
) as llm:
|
||||
if model_name == GRANITE_MODEL_NAME:
|
||||
runner_kwargs = {
|
||||
"quantization": "fp8_per_tensor",
|
||||
"dtype": "bfloat16",
|
||||
"enforce_eager": True,
|
||||
"hf_overrides": {"num_hidden_layers": 3},
|
||||
"max_model_len": 16,
|
||||
"max_num_seqs": 1,
|
||||
}
|
||||
else:
|
||||
runner_kwargs = {
|
||||
"quantization_config": {
|
||||
"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}
|
||||
},
|
||||
"enforce_eager": True,
|
||||
}
|
||||
|
||||
def check_model(model):
|
||||
layer = model.model.layers[0]
|
||||
assert isinstance(
|
||||
layer.self_attn.o_proj.quant_method,
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
layer.block_sparse_moe.experts._quant_method,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
)
|
||||
with vllm_runner(model_name, **runner_kwargs) as llm:
|
||||
if model_name == GRANITE_MODEL_NAME:
|
||||
|
||||
def check_model(model):
|
||||
layer = model.model.layers[0]
|
||||
assert isinstance(
|
||||
layer.self_attn.o_proj.quant_method,
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
layer.block_sparse_moe.experts._quant_method,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
def check_model(model):
|
||||
assert isinstance(
|
||||
model.model.layers[1].self_attn.o_proj.quant_method,
|
||||
Mxfp8OnlineLinearMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
model.model.layers[0].self_attn.o_proj.quant_method,
|
||||
CompressedTensorsLinearMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
model.model.layers[0].mlp.experts._quant_method,
|
||||
CompressedTensorsMoEMethod,
|
||||
)
|
||||
|
||||
llm.apply_model(check_model)
|
||||
outputs = llm.generate_greedy(["Hello my name is"], max_tokens=1)
|
||||
@@ -457,8 +917,8 @@ def test_online_quantization_targets_ignore_collision() -> None:
|
||||
)
|
||||
)
|
||||
with pytest.raises(ValueError, match="matches both quantization_config.ignore"):
|
||||
config._dispatch_target(
|
||||
"model.layers.0.self_attn.o_proj", Mock(spec=LinearBase)
|
||||
config.resolve_quant_method_cls(
|
||||
Mock(spec=LinearBase), "model.layers.0.self_attn.o_proj"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -354,7 +354,7 @@ def test_online_mxfp4_moe_matches_quark(
|
||||
weight_quant_key=kMxfp4Static,
|
||||
activation_quant_key=kMxfp4Dynamic,
|
||||
)
|
||||
online_method = Mxfp4OnlineMoEMethod(layer=online_layer)
|
||||
online_method = Mxfp4OnlineMoEMethod(moe=online_layer.moe_config)
|
||||
|
||||
# `RoutedExperts.__init__` applies this round-up in production; these
|
||||
# layers are built without a quant config, so it is applied explicitly.
|
||||
|
||||
@@ -95,6 +95,12 @@ def test_resolve_shorthand_only_populates_both_slots():
|
||||
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("quantization", ["mxfp4", "mxfp8"])
|
||||
def test_resolve_colliding_shorthand_is_deferred(quantization: str):
|
||||
"""Checkpoint metadata determines whether an MXFP shorthand is online."""
|
||||
assert resolve_quantization_config(quantization, None) is None
|
||||
|
||||
|
||||
def test_resolve_int8_shorthand_leaves_linear_unset():
|
||||
# int8_per_channel_weight_only is MoE-only; linear stays None so that
|
||||
# OnlineQuantizationConfig leaves Linear layers in full precision.
|
||||
@@ -122,12 +128,9 @@ def test_resolve_merges_explicit_over_shorthand():
|
||||
assert args.moe == QuantSpec(weight=kFp8StaticTensorSym)
|
||||
|
||||
|
||||
def test_resolve_rejects_quantization_config_with_non_shorthand_quant():
|
||||
# If --quantization names something other than an online shorthand,
|
||||
# quantization_config is not allowed via this path (checkpoint quant
|
||||
# paths read it directly off ModelConfig instead).
|
||||
with pytest.raises(ValueError, match="quantization_config is only supported"):
|
||||
resolve_quantization_config("gptq", {"linear": "fp8_per_block"})
|
||||
def test_resolve_quantization_config_with_checkpoint_quantization():
|
||||
args = resolve_quantization_config("gptq", {"linear": "fp8_per_block"})
|
||||
assert args == QuantizationConfigArgs(linear="fp8_per_block")
|
||||
|
||||
|
||||
# ---- QUANT_KEY_NAMES coverage --------------------------------------------
|
||||
@@ -206,6 +209,6 @@ def test_targets_reject_moe_only_shorthand_for_linear_layer():
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not define a QuantSpec"):
|
||||
config._dispatch_target(
|
||||
"model.layers.0.self_attn.o_proj", Mock(spec=LinearBase)
|
||||
config.resolve_quant_method_cls(
|
||||
Mock(spec=LinearBase), "model.layers.0.self_attn.o_proj"
|
||||
)
|
||||
|
||||
@@ -225,6 +225,10 @@ ONLINE_QUANT_SHORTHAND_NAMES: tuple[str, ...] = (
|
||||
"online",
|
||||
)
|
||||
|
||||
# These names are also checkpoint quantization methods. Their online configs
|
||||
# are resolved only when checkpoint quantization metadata is absent.
|
||||
_DEFERRED_ONLINE_SHORTHANDS = frozenset(("mxfp4", "mxfp8"))
|
||||
|
||||
|
||||
def resolve_quantization_config(
|
||||
quantization: str | None,
|
||||
@@ -239,17 +243,25 @@ def resolve_quantization_config(
|
||||
take precedence over the shorthand.
|
||||
"""
|
||||
if quantization is not None and quantization not in ONLINE_QUANT_SHORTHAND_NAMES:
|
||||
if quantization_config is not None:
|
||||
raise ValueError(
|
||||
f"quantization_config is only supported when quantization is "
|
||||
f"one of {sorted(ONLINE_QUANT_SHORTHAND_NAMES)}, "
|
||||
f"got quantization={quantization!r}"
|
||||
)
|
||||
return None
|
||||
# Pre-quantized checkpoints can be composed with online quantization
|
||||
# for layers that the base quant_method leaves unquantized. The
|
||||
# checkpoint quant_method remains the primary quantization method; composition
|
||||
# is performed after its config has been loaded.
|
||||
if quantization_config is None:
|
||||
return None
|
||||
|
||||
# `quantization_config` may hold both:
|
||||
# 1. Base quantization method activation key override,
|
||||
# 2. online quantization config to apply on top of the base quant_method.
|
||||
if isinstance(quantization_config, dict):
|
||||
return QuantizationConfigArgs(**quantization_config)
|
||||
return quantization_config
|
||||
|
||||
base = _ONLINE_SHORTHANDS.get(quantization) if quantization else None
|
||||
|
||||
if quantization_config is None:
|
||||
if quantization in _DEFERRED_ONLINE_SHORTHANDS:
|
||||
return None
|
||||
return base
|
||||
|
||||
if isinstance(quantization_config, dict):
|
||||
|
||||
@@ -19,7 +19,10 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.linear import (
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
QuantizationConfig,
|
||||
resolve_quant_method,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
@@ -183,7 +186,9 @@ def _init_kv_cache_quant(
|
||||
layer._o_scale_float = None
|
||||
|
||||
quant_method = (
|
||||
quant_config.get_quant_method(layer, prefix=prefix) if quant_config else None
|
||||
resolve_quant_method(quant_config, layer, prefix=prefix)
|
||||
if quant_config
|
||||
else None
|
||||
)
|
||||
|
||||
# See [Note: Register q/k/v/prob scales in state dict]
|
||||
@@ -584,7 +589,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
# as the default value. See [Note: Register q/k/v/prob scales in state dict]
|
||||
# for more details.
|
||||
quant_method = (
|
||||
self.quant_config.get_quant_method(self, prefix=self.layer_name)
|
||||
resolve_quant_method(self.quant_config, self, prefix=self.layer_name)
|
||||
if self.quant_config
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -246,7 +246,10 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
QuantizationConfig,
|
||||
resolve_quant_method,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
@@ -1255,7 +1258,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
# as the default value. See [Note: Register q/k/v/prob scales in state dict]
|
||||
# for more details.
|
||||
quant_method = (
|
||||
self.quant_config.get_quant_method(self, prefix=self.layer_name)
|
||||
resolve_quant_method(self.quant_config, self, prefix=self.layer_name)
|
||||
if self.quant_config
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
resolve_quant_method,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -199,7 +200,7 @@ class RoutedExperts(PluggableLayer):
|
||||
"""
|
||||
quant_method = None
|
||||
if quant_config is not None:
|
||||
quant_method = quant_config.get_quant_method(self, prefix)
|
||||
quant_method = resolve_quant_method(quant_config, self, prefix)
|
||||
if quant_method is None:
|
||||
quant_method = UnquantizedFusedMoEMethod(moe_config)
|
||||
assert isinstance(quant_method, FusedMoEMethodBase)
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.determinism.batch_invariant import (
|
||||
linear_batch_invariant,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import resolve_quant_method
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
@@ -282,7 +283,7 @@ class LinearBase(PluggableLayer):
|
||||
self.quant_method: QuantizeMethodBase
|
||||
if quant_config is None:
|
||||
self.quant_method = UnquantizedLinearMethod()
|
||||
elif quant_method := quant_config.get_quant_method(self, prefix=prefix):
|
||||
elif quant_method := resolve_quant_method(quant_config, self, prefix=prefix):
|
||||
self.quant_method = quant_method
|
||||
else:
|
||||
raise ValueError("All linear layers should support quant method.")
|
||||
|
||||
@@ -4,7 +4,10 @@
|
||||
from typing import Literal, get_args
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
resolve_quant_method,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -182,6 +185,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
|
||||
__all__ = [
|
||||
"QuantizationConfig",
|
||||
"resolve_quant_method",
|
||||
"QuantizationMethods",
|
||||
"get_quantization_config",
|
||||
"register_quantization_config",
|
||||
|
||||
@@ -12,6 +12,9 @@ from transformers import PretrainedConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
else:
|
||||
QuantizationMethods = str
|
||||
@@ -105,6 +108,7 @@ class QuantizationConfig(ABC):
|
||||
"""Suffixes of quantization parameters that may be present in the checkpoint but
|
||||
not in the model, and should be ignored if unexpected during loading. These are used
|
||||
after remapping, so should be in vLLM format (e.g. .q_scale, not .q.scale)."""
|
||||
online_quantization_config: "OnlineQuantizationConfig | None" = None
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -187,7 +191,8 @@ class QuantizationConfig(ABC):
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> QuantizeMethodBase | None:
|
||||
"""Get the quantize method to use for the quantized layer.
|
||||
"""Get the quantize method to use for the quantized layer, from the
|
||||
pre-quantized checkpoint quant_method.
|
||||
|
||||
Args:
|
||||
layer: The layer for the quant method.
|
||||
@@ -265,3 +270,46 @@ class QuantizationConfig(ABC):
|
||||
# TODO: revision is never passed currently in vllm.py,
|
||||
# but is used in subclasses, should we remove this parameter?
|
||||
pass
|
||||
|
||||
|
||||
def resolve_quant_method(
|
||||
quant_config: QuantizationConfig, layer: torch.nn.Module, prefix: str
|
||||
) -> QuantizeMethodBase | None:
|
||||
"""Return the checkpoint method with configured online quantization."""
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
|
||||
base_quant_method = quant_config.get_quant_method(layer, prefix)
|
||||
if quant_config.online_quantization_config is None:
|
||||
return base_quant_method
|
||||
# Online quantization currently supports only LinearBase and RoutedExperts.
|
||||
# Embeddings and ParallelLMHead retain their checkpoint quantization method.
|
||||
if not isinstance(layer, (LinearBase, RoutedExperts)):
|
||||
return base_quant_method
|
||||
|
||||
quant_config.online_quantization_config.packed_modules_mapping = (
|
||||
quant_config.packed_modules_mapping
|
||||
)
|
||||
checkpoint_is_quantized = base_quant_method is not None and not isinstance(
|
||||
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
|
||||
)
|
||||
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
|
||||
layer, prefix
|
||||
)
|
||||
if checkpoint_is_quantized:
|
||||
if online_target is not None:
|
||||
raise ValueError(
|
||||
f"Cannot apply requested online quantization {online_target[3]} to "
|
||||
f"pre-quantized layer {prefix}: {base_quant_method} was already "
|
||||
"selected by the checkpoint quantization config."
|
||||
)
|
||||
return base_quant_method
|
||||
if online_target is None:
|
||||
return base_quant_method
|
||||
return quant_config.online_quantization_config.get_quant_method(layer, prefix)
|
||||
|
||||
@@ -56,5 +56,5 @@ class ExpertsInt8Config(QuantizationConfig):
|
||||
if isinstance(layer, LinearBase):
|
||||
return UnquantizedLinearMethod()
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
return Int8OnlineMoEMethod(layer=layer)
|
||||
return Int8OnlineMoEMethod(moe=layer.moe_config)
|
||||
return None
|
||||
|
||||
@@ -219,7 +219,7 @@ class Fp8Config(QuantizationConfig):
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
)
|
||||
|
||||
return Fp8PerTensorOnlineMoEMethod(layer=layer)
|
||||
return Fp8PerTensorOnlineMoEMethod(moe=layer.moe_config)
|
||||
elif isinstance(layer, Attention):
|
||||
return Fp8KVCacheMethod(self)
|
||||
return None
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Mapping
|
||||
from enum import Enum
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
@@ -14,6 +15,7 @@ from vllm.config.quantization import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEMethodBase,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
@@ -38,6 +40,7 @@ from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
Fp8PtpcOnlineLinearMethod,
|
||||
Fp8PtpcOnlineMoEMethod,
|
||||
OnlineLinearBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.int8 import (
|
||||
Int8OnlineMoEMethod,
|
||||
@@ -71,6 +74,14 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OnlineQuantizationSource(str, Enum):
|
||||
"""Supported online quantization configuration sources."""
|
||||
|
||||
linear = "linear" # LinearBase
|
||||
moe = "moe" # RoutedExperts
|
||||
targets = "targets"
|
||||
|
||||
|
||||
# Online dispatch tables, keyed by the QuantSpec.weight QuantKey. The
|
||||
# corresponding method class handles the activation choice via its
|
||||
# `supported_activation_quant` set.
|
||||
@@ -200,12 +211,23 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
"quantization='fp8_per_tensor'/'fp8_per_block' instead."
|
||||
)
|
||||
|
||||
def _dispatch(
|
||||
def _get_method_cls(
|
||||
self,
|
||||
spec: QuantSpec | None,
|
||||
table: dict[QuantKey, type],
|
||||
layer: torch.nn.Module,
|
||||
) -> "QuantizeMethodBase | None":
|
||||
) -> type | None:
|
||||
"""Resolve the online method class for a layer's quantization spec.
|
||||
|
||||
Args:
|
||||
spec: Quantization specification to resolve.
|
||||
table: Mapping from weight quantization keys to method classes.
|
||||
layer: Layer that will use the resolved method.
|
||||
|
||||
Returns:
|
||||
The matching method class, or None when ``spec`` has no weight
|
||||
quantization.
|
||||
"""
|
||||
if spec is None or spec.weight is None:
|
||||
return None
|
||||
cls = table.get(spec.weight)
|
||||
@@ -223,13 +245,75 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
f"activation override (activation={spec.activation}) is not "
|
||||
f"yet supported for online {cls.__name__}"
|
||||
)
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return cls(layer=layer)
|
||||
return cls()
|
||||
return cls
|
||||
|
||||
def _dispatch_target(
|
||||
def resolve_quant_method_cls(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> tuple[OnlineQuantizationSource, str, str | None, QuantSpec, type] | None:
|
||||
"""Resolve quantization metadata and method class without instantiating it.
|
||||
|
||||
Args:
|
||||
layer: Layer for which to resolve online quantization.
|
||||
prefix: Fully qualified layer name.
|
||||
|
||||
Returns:
|
||||
A tuple of source, quantization key string, target pattern, spec,
|
||||
and method class. Returns None when online quantization does not
|
||||
apply to the layer.
|
||||
"""
|
||||
quant_spec: QuantSpec | None
|
||||
if self.args.targets is not None:
|
||||
resolved_pattern = self._resolve_targets_quant_method_metadata(
|
||||
prefix, layer
|
||||
)
|
||||
if resolved_pattern is None:
|
||||
return None
|
||||
source, quant_key_str, target_pattern, quant_spec, table = resolved_pattern
|
||||
else:
|
||||
if isinstance(layer, LinearBase):
|
||||
source = OnlineQuantizationSource.linear
|
||||
quant_spec = self.args.linear
|
||||
table = _ONLINE_LINEAR_METHODS
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
source = OnlineQuantizationSource.moe
|
||||
quant_spec = self.args.moe
|
||||
table = _ONLINE_MOE_METHODS
|
||||
else:
|
||||
return None
|
||||
|
||||
if should_ignore_layer(
|
||||
prefix,
|
||||
ignore=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
use_fnmatch=True,
|
||||
):
|
||||
return None
|
||||
quant_key_str = str(quant_spec)
|
||||
target_pattern = None
|
||||
|
||||
quant_method_cls = self._get_method_cls(quant_spec, table, layer)
|
||||
if quant_method_cls is None:
|
||||
return None
|
||||
assert quant_spec is not None
|
||||
return source, quant_key_str, target_pattern, quant_spec, quant_method_cls
|
||||
|
||||
def _resolve_targets_quant_method_metadata(
|
||||
self, prefix: str, layer: torch.nn.Module
|
||||
) -> "QuantizeMethodBase | None":
|
||||
) -> (
|
||||
tuple[OnlineQuantizationSource, str, str, QuantSpec, dict[QuantKey, type]]
|
||||
| None
|
||||
):
|
||||
"""Resolve target-pattern quantization metadata and dispatch table.
|
||||
|
||||
Args:
|
||||
prefix: Fully qualified layer name.
|
||||
layer: Layer matched against configured target patterns.
|
||||
|
||||
Returns:
|
||||
A tuple of source, quantization key string, target pattern, spec,
|
||||
and dispatch table. Returns None when no pattern applies or the
|
||||
layer is ignored.
|
||||
"""
|
||||
assert self.args.targets is not None
|
||||
ignored = should_ignore_layer(
|
||||
prefix,
|
||||
@@ -254,7 +338,8 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
f"targets patterns: {matches}. Each layer may match at most "
|
||||
f"one target."
|
||||
)
|
||||
quant_key_str = self.args.targets[matches[0]]
|
||||
target_pattern = matches[0]
|
||||
quant_key_str = self.args.targets[target_pattern]
|
||||
shorthand = _ONLINE_SHORTHANDS[quant_key_str]
|
||||
if isinstance(layer, LinearBase):
|
||||
quant_spec = shorthand.linear
|
||||
@@ -265,63 +350,44 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Layer {prefix} was matched by quantization_config.targets "
|
||||
f"({matches[0]}), but online quantization is not supported for "
|
||||
f"({target_pattern}), but online quantization is not supported for "
|
||||
f"{type(layer).__name__}."
|
||||
)
|
||||
if quant_spec is None:
|
||||
raise ValueError(
|
||||
f"targets pattern {matches[0]} = {quant_key_str} does "
|
||||
f"targets pattern {target_pattern} = {quant_key_str} does "
|
||||
f"not define a QuantSpec for {type(layer).__name__} layers "
|
||||
f"(matched at {prefix})."
|
||||
)
|
||||
method = self._dispatch(quant_spec, table, layer)
|
||||
if method is not None:
|
||||
self.quantized_layers[prefix] = ("targets", quant_key_str, matches[0])
|
||||
return method
|
||||
return (
|
||||
OnlineQuantizationSource.targets,
|
||||
quant_key_str,
|
||||
target_pattern,
|
||||
quant_spec,
|
||||
table,
|
||||
)
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
# `targets` takes precedence over `moe` and `linear` and is exclusive.
|
||||
if self.args.targets is not None:
|
||||
method = self._dispatch_target(prefix, layer)
|
||||
if method is not None:
|
||||
return method
|
||||
|
||||
if isinstance(layer, LinearBase):
|
||||
return UnquantizedLinearMethod()
|
||||
resolved = self.resolve_quant_method_cls(layer, prefix)
|
||||
if resolved is not None:
|
||||
source, quant_key_str, target_pattern, _, quant_method_cls = resolved
|
||||
self.quantized_layers[prefix] = (
|
||||
source.value,
|
||||
quant_key_str,
|
||||
target_pattern,
|
||||
)
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
return None
|
||||
assert issubclass(quant_method_cls, FusedMoEMethodBase)
|
||||
return quant_method_cls(moe=layer.moe_config)
|
||||
|
||||
assert issubclass(quant_method_cls, OnlineLinearBase)
|
||||
return quant_method_cls()
|
||||
|
||||
if isinstance(layer, LinearBase):
|
||||
if should_ignore_layer(
|
||||
prefix,
|
||||
ignore=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
use_fnmatch=True,
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer)
|
||||
if method is not None:
|
||||
self.quantized_layers[prefix] = (
|
||||
"linear",
|
||||
str(self.args.linear),
|
||||
None,
|
||||
)
|
||||
return method
|
||||
return UnquantizedLinearMethod()
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
if should_ignore_layer(
|
||||
prefix,
|
||||
ignore=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
use_fnmatch=True,
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer)
|
||||
if method is not None:
|
||||
self.quantized_layers[prefix] = ("moe", str(self.args.moe), None)
|
||||
return method
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
return None
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
select_fp8_moe_backend,
|
||||
)
|
||||
@@ -111,7 +112,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
|
||||
return is_row_parallel or (reduces_output_dim and is_column_parallel)
|
||||
|
||||
|
||||
class _Fp8OnlineLinearBase(LinearMethodBase):
|
||||
class OnlineLinearBase(LinearMethodBase):
|
||||
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
|
||||
weights onto meta device and materializes them just-in-time."""
|
||||
|
||||
@@ -155,7 +156,7 @@ class _Fp8OnlineLinearBase(LinearMethodBase):
|
||||
initialize_online_processing(layer)
|
||||
|
||||
|
||||
class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online tensorwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
|
||||
|
||||
@@ -256,7 +257,7 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
|
||||
class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online blockwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading."""
|
||||
|
||||
@@ -336,7 +337,7 @@ class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
)
|
||||
|
||||
|
||||
class Fp8PtpcOnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online PTPC FP8 linear quantization.
|
||||
|
||||
Per-output-channel weight scale + dynamic per-token activation scale. The
|
||||
@@ -443,12 +444,12 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
self,
|
||||
*,
|
||||
weight_block_size: list[int] | None,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
weight_key: "QuantKey | None" = None,
|
||||
activation_key: "QuantKey | None" = None,
|
||||
allow_vllm_cutlass: bool = False,
|
||||
):
|
||||
super().__init__(layer.moe_config)
|
||||
super().__init__(moe)
|
||||
self.weight_block_size = weight_block_size
|
||||
self.block_quant: bool = self.weight_block_size is not None
|
||||
self.weight_scale_name = (
|
||||
@@ -555,11 +556,11 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
super().__init__(
|
||||
weight_block_size=None,
|
||||
layer=layer,
|
||||
moe=moe,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
@@ -612,11 +613,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
super().__init__(
|
||||
weight_block_size=[128, 128],
|
||||
layer=layer,
|
||||
moe=moe,
|
||||
)
|
||||
|
||||
def maybe_roundup_sizes(
|
||||
@@ -716,13 +717,13 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
|
||||
|
||||
super().__init__(
|
||||
weight_block_size=None,
|
||||
layer=layer,
|
||||
moe=moe,
|
||||
weight_key=kFp8StaticChannelSym,
|
||||
activation_key=kFp8DynamicTokenSym,
|
||||
allow_vllm_cutlass=True,
|
||||
|
||||
@@ -12,6 +12,7 @@ if TYPE_CHECKING:
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
|
||||
convert_to_int8_moe_kernel_format,
|
||||
make_int8_moe_kernel,
|
||||
@@ -38,9 +39,9 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
super().__init__(layer.moe_config)
|
||||
super().__init__(moe)
|
||||
self.int8_backend, self.experts_cls = select_int8_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=kInt8StaticChannelSym,
|
||||
|
||||
@@ -17,6 +17,7 @@ if TYPE_CHECKING:
|
||||
)
|
||||
|
||||
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
TRITON_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
@@ -27,7 +28,7 @@ from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
select_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
_Fp8OnlineLinearBase,
|
||||
OnlineLinearBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
@@ -73,7 +74,7 @@ def _quantize_mxfp4_moe_weight(
|
||||
return w_quant, w_scales
|
||||
|
||||
|
||||
class Mxfp4OnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
class Mxfp4OnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online MXFP4 linear method.
|
||||
Loads bf16/fp16 checkpoints and quantizes weights to MXFP4 (microscaling
|
||||
FP4 with block-32 scales) during weight loading.
|
||||
@@ -144,8 +145,8 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
experts_cls: "type[mk.FusedMoEExperts] | None"
|
||||
activation_quant_key = kMxfp4Dynamic
|
||||
|
||||
def __init__(self, *, layer: torch.nn.Module):
|
||||
super().__init__(layer.moe_config)
|
||||
def __init__(self, *, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.weight_block_size: list[int] = [1, MXFP4_BLOCK_SIZE]
|
||||
self.weight_scale_name = "weight_scale"
|
||||
|
||||
|
||||
@@ -8,6 +8,8 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
from torch.nn import Module
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
@@ -21,7 +23,7 @@ from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
_Fp8OnlineLinearBase,
|
||||
OnlineLinearBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
@@ -34,7 +36,7 @@ from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
class Mxfp8OnlineLinearMethod(_Fp8OnlineLinearBase):
|
||||
class Mxfp8OnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online MXFP8 linear method.
|
||||
Loads bf16/fp16 checkpoints and quantizes weights to MXFP8 (microscaling
|
||||
FP8 with block-32 scales) during weight loading.
|
||||
@@ -100,8 +102,8 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
fp8_backend: "Fp8MoeBackend"
|
||||
experts_cls: "type[mk.FusedMoEExperts] | None"
|
||||
|
||||
def __init__(self, *, layer: torch.nn.Module):
|
||||
super().__init__(layer.moe_config)
|
||||
def __init__(self, *, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.weight_block_size: list[int] = [1, MXFP8_BLOCK_SIZE]
|
||||
self.weight_scale_name = "weight_scale"
|
||||
|
||||
|
||||
@@ -6,7 +6,10 @@ from torch.nn import Module
|
||||
|
||||
from vllm._custom_ops import scaled_fp4_quant
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
convert_to_nvfp4_moe_kernel_format,
|
||||
make_nvfp4_moe_kernel,
|
||||
@@ -87,13 +90,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
layer: torch.nn.Module,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
raise ValueError(
|
||||
"nvfp4_per_token online quantization requires a Blackwell (SM100) GPU."
|
||||
)
|
||||
super().__init__(layer.moe_config)
|
||||
super().__init__(moe)
|
||||
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=kNvfp4Static,
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
method_has_implemented_embedding,
|
||||
resolve_quant_method,
|
||||
)
|
||||
from vllm.model_executor.layers.utils import dispatch_unquantized_gemm
|
||||
from vllm.model_executor.parameter import BasevLLMParameter
|
||||
@@ -291,7 +292,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
# Avoid overriding a preselected model-specific method with generic
|
||||
# config-based dispatch.
|
||||
if quant_method is None and quant_config is not None:
|
||||
quant_method = quant_config.get_quant_method(self, prefix=prefix)
|
||||
quant_method = resolve_quant_method(quant_config, self, prefix=prefix)
|
||||
if quant_method is None:
|
||||
quant_method = UnquantizedEmbeddingMethod()
|
||||
|
||||
|
||||
@@ -100,6 +100,11 @@ def log_online_quantization(vllm_config: VllmConfig) -> None:
|
||||
)
|
||||
|
||||
quant_config = vllm_config.quant_config
|
||||
online_quantization_config = getattr(
|
||||
quant_config, "online_quantization_config", None
|
||||
)
|
||||
if isinstance(online_quantization_config, OnlineQuantizationConfig):
|
||||
quant_config = online_quantization_config
|
||||
if not isinstance(quant_config, OnlineQuantizationConfig):
|
||||
return
|
||||
|
||||
|
||||
@@ -243,6 +243,26 @@ def get_quant_config(
|
||||
if model_config.quantization is None:
|
||||
raise ValueError("Model quantization method is not specified in the config.")
|
||||
quant_cls = get_quantization_config(model_config.quantization)
|
||||
from vllm.config.quantization import _ONLINE_SHORTHANDS, QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
|
||||
# resolve_quantization_config does not resolve `quantization_config` from
|
||||
# ambiguous shorthands as `"mxfp4"`, `"mxfp8"`.
|
||||
online_args = model_config.quantization_config
|
||||
|
||||
def maybe_compose_online_quantization(
|
||||
checkpoint_config: QuantizationConfig,
|
||||
) -> QuantizationConfig:
|
||||
if online_args is None:
|
||||
return checkpoint_config
|
||||
assert isinstance(online_args, QuantizationConfigArgs)
|
||||
|
||||
checkpoint_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
online_args
|
||||
)
|
||||
return checkpoint_config
|
||||
|
||||
# Read the quantization config from the HF model config, if available.
|
||||
hf_quant_config = getattr(model_config.hf_config, "quantization_config", None)
|
||||
@@ -284,7 +304,9 @@ def get_quant_config(
|
||||
):
|
||||
pass # fall through to file-based loading below
|
||||
else:
|
||||
return quant_cls.from_config(hf_quant_config)
|
||||
return maybe_compose_online_quantization(
|
||||
quant_cls.from_config(hf_quant_config)
|
||||
)
|
||||
|
||||
# if hf_quant_config is None, we will try to get config from
|
||||
# hf_overrides
|
||||
@@ -302,7 +324,9 @@ def get_quant_config(
|
||||
quantization_config_file = hf_overrides.get("quantization_config_file")
|
||||
if quantization_config_file is not None:
|
||||
if hasattr(quant_cls, "from_config_file"):
|
||||
return quant_cls.from_config_file(quantization_config_file)
|
||||
return maybe_compose_online_quantization(
|
||||
quant_cls.from_config_file(quantization_config_file)
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"from_config_file is specified in hf_override config, "
|
||||
@@ -312,7 +336,9 @@ def get_quant_config(
|
||||
quantization_config_json = hf_overrides.get("quantization_config_dict_json")
|
||||
if quantization_config_json is not None:
|
||||
if hasattr(quant_cls, "from_config_dict_json"):
|
||||
return quant_cls.from_config_dict_json(quantization_config_json)
|
||||
return maybe_compose_online_quantization(
|
||||
quant_cls.from_config_dict_json(quantization_config_json)
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"from_config_dict_json is specified in hf_override config, "
|
||||
@@ -320,16 +346,14 @@ def get_quant_config(
|
||||
f"{quant_cls}"
|
||||
)
|
||||
|
||||
# Online quantization doesn't read from checkpoint configs - it quantizes
|
||||
# fp16/bf16 weights on the fly during loading.
|
||||
if model_config.quantization_config is not None:
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
|
||||
assert isinstance(model_config.quantization_config, QuantizationConfigArgs)
|
||||
return OnlineQuantizationConfig(args=model_config.quantization_config)
|
||||
# Raw unambiguous online quantization doesn't read from checkpoint configs
|
||||
# We must continue to load/read the config below in two cases:
|
||||
# 1. composed online quantization, before `maybe_compose_online_quantization`,
|
||||
# 2. Ambiguous online shorthands as `"mxpf4"`, `"mxfp8"`, for which `online_args`
|
||||
# is not set yet.
|
||||
if quant_cls is OnlineQuantizationConfig and online_args is not None:
|
||||
assert isinstance(online_args, QuantizationConfigArgs)
|
||||
return OnlineQuantizationConfig(args=online_args)
|
||||
|
||||
model_name_or_path = (
|
||||
maybe_download_from_modelscope(
|
||||
@@ -359,7 +383,11 @@ def get_quant_config(
|
||||
|
||||
# If the quantization config is not found, use the default config.
|
||||
if not possible_config_filenames:
|
||||
return quant_cls()
|
||||
if model_config.quantization in _ONLINE_SHORTHANDS:
|
||||
args = online_args or _ONLINE_SHORTHANDS[model_config.quantization]
|
||||
assert isinstance(args, QuantizationConfigArgs)
|
||||
return OnlineQuantizationConfig(args=args)
|
||||
return maybe_compose_online_quantization(quant_cls())
|
||||
|
||||
config_files = glob.glob(os.path.join(hf_folder, "*.json"))
|
||||
|
||||
@@ -367,6 +395,10 @@ def get_quant_config(
|
||||
f for f in config_files if any(f.endswith(x) for x in possible_config_filenames)
|
||||
]
|
||||
if len(quant_config_files) == 0:
|
||||
if model_config.quantization in _ONLINE_SHORTHANDS:
|
||||
args = online_args or _ONLINE_SHORTHANDS[model_config.quantization]
|
||||
assert isinstance(args, QuantizationConfigArgs)
|
||||
return OnlineQuantizationConfig(args=args)
|
||||
raise ValueError(f"Cannot find the config file for {model_config.quantization}")
|
||||
if len(quant_config_files) > 1:
|
||||
raise ValueError(
|
||||
@@ -380,14 +412,14 @@ def get_quant_config(
|
||||
|
||||
if model_config.quantization in ("modelopt", "modelopt_mixed"):
|
||||
if config.get("producer", {}).get("name") == "modelopt":
|
||||
return quant_cls.from_config(config)
|
||||
return maybe_compose_online_quantization(quant_cls.from_config(config))
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported quantization config"
|
||||
f" found for {model_config.quantization} in {f}."
|
||||
)
|
||||
|
||||
return quant_cls.from_config(config)
|
||||
return maybe_compose_online_quantization(quant_cls.from_config(config))
|
||||
|
||||
|
||||
def get_sparse_attention_config(
|
||||
|
||||
@@ -69,7 +69,10 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
QuantizationConfig,
|
||||
resolve_quant_method,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_and_maybe_dequant_weights,
|
||||
)
|
||||
@@ -462,7 +465,7 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
|
||||
|
||||
quant_method = (
|
||||
self.quant_config.get_quant_method(self, prefix=self.layer_name)
|
||||
resolve_quant_method(self.quant_config, self, prefix=self.layer_name)
|
||||
if self.quant_config
|
||||
else None
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user