[Quantization] Support online quantization with partially pre-quantized checkpoints (#51392)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
a11dfcff89
commit
5690b02c03
@@ -58,7 +58,7 @@ def test_online_ignore_supports_fnmatch_patterns():
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="matches both quantization_config.ignore"):
|
||||
config._dispatch_target(layer_name, Mock())
|
||||
config.resolve_quant_method_cls(Mock(), layer_name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests online quantization."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
from typing import Any, cast
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
@@ -18,18 +20,39 @@ from tests.quantization.utils import (
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm._custom_ops import scaled_fp4_quant
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config.quantization import (
|
||||
QuantizationConfigArgs,
|
||||
resolve_quantization_config,
|
||||
)
|
||||
from vllm.config.vllm import VllmConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.kernels.linear.mxfp8.emulation import (
|
||||
EmulationMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.marlin import (
|
||||
MarlinMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
||||
CompressedTensorsConfig,
|
||||
CompressedTensorsLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
|
||||
CompressedTensorsMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptFp8Config,
|
||||
ModelOptMxFp8Config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
@@ -50,19 +73,31 @@ from vllm.model_executor.layers.quantization.online.mxfp4 import (
|
||||
Mxfp4OnlineLinearMethod,
|
||||
Mxfp4OnlineMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.mxfp8 import (
|
||||
Mxfp8OnlineLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.nvfp4 import (
|
||||
Nvfp4OnlineMoEMethod,
|
||||
_quantize_moe_weight_to_nvfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.quark import (
|
||||
QuarkConfig,
|
||||
QuarkLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils import quant_utils
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_VALUE_DTYPE,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
amax_for_moe_weight_quant,
|
||||
amax_for_tp_weight_quant,
|
||||
kMxfp8Dynamic,
|
||||
weight_amax,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader import weight_utils
|
||||
from vllm.model_executor.model_loader.base_loader import log_online_quantization
|
||||
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
|
||||
from vllm.model_executor.models.granitemoe import (
|
||||
@@ -83,6 +118,10 @@ else:
|
||||
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
GRANITE_MODEL_NAME = "ibm-granite/granite-3.0-1b-a400m-base"
|
||||
PARTIALLY_PREQUANTIZED_MODEL_NAME = (
|
||||
"nm-testing/tinysmokeqwen3moe-W4A16-first-only-CTstable"
|
||||
)
|
||||
|
||||
|
||||
def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
@@ -132,15 +171,349 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
assert process_weights.call_count == 2
|
||||
|
||||
|
||||
def _fully_quantized_quark_config() -> QuarkConfig:
|
||||
return QuarkConfig(
|
||||
{
|
||||
"exclude": [],
|
||||
"global_quant_config": {
|
||||
"weight": {
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
"input_tensors": {
|
||||
"dtype": "int8",
|
||||
"qscheme": "per_tensor",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
},
|
||||
},
|
||||
"layer_quant_config": {},
|
||||
"layer_type_quant_config": {},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _fully_quantized_modelopt_config() -> ModelOptFp8Config:
|
||||
return ModelOptFp8Config(
|
||||
quant_method="FP8",
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
kv_cache_quant_method=None,
|
||||
exclude_modules=[],
|
||||
)
|
||||
|
||||
|
||||
def _moe_only_compressed_tensors_config() -> CompressedTensorsConfig:
|
||||
return CompressedTensorsConfig(
|
||||
target_scheme_map={"RoutedExperts": {}},
|
||||
ignore=[],
|
||||
quant_format="pack-quantized",
|
||||
)
|
||||
|
||||
|
||||
def _write_minimal_llama_config(
|
||||
model_path: Path, quantization_config: dict[str, Any] | None = None
|
||||
) -> None:
|
||||
config: dict[str, Any] = {
|
||||
"architectures": ["LlamaForCausalLM"],
|
||||
"model_type": "llama",
|
||||
"hidden_size": 32,
|
||||
"intermediate_size": 64,
|
||||
"num_attention_heads": 4,
|
||||
"num_hidden_layers": 1,
|
||||
"vocab_size": 32,
|
||||
"max_position_embeddings": 32,
|
||||
}
|
||||
if quantization_config is not None:
|
||||
config["quantization_config"] = quantization_config
|
||||
(model_path / "config.json").write_text(json.dumps(config))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"checkpoint_config_factory,raises_conflict",
|
||||
[
|
||||
pytest.param(_fully_quantized_quark_config, True, id="quark"),
|
||||
pytest.param(_fully_quantized_modelopt_config, True, id="modelopt"),
|
||||
pytest.param(
|
||||
_moe_only_compressed_tensors_config,
|
||||
False,
|
||||
id="compressed_tensors",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_online_prequantized_compatibility(
|
||||
checkpoint_config_factory,
|
||||
raises_conflict: bool,
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
) -> None:
|
||||
"""Online weights replace only layers left unquantized by a checkpoint."""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
checkpoint_config = checkpoint_config_factory()
|
||||
|
||||
checkpoint_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(linear="mxfp8")
|
||||
)
|
||||
config = checkpoint_config
|
||||
|
||||
layer_kwargs = {
|
||||
"input_size": 32,
|
||||
"output_size": 32,
|
||||
"bias": False,
|
||||
"params_dtype": torch.bfloat16,
|
||||
"quant_config": config,
|
||||
"prefix": "model.layers.0.self_attn.o_proj",
|
||||
"disable_tp": True,
|
||||
}
|
||||
|
||||
if raises_conflict:
|
||||
with pytest.raises(ValueError, match="pre-quantized layer"):
|
||||
ColumnParallelLinear(**layer_kwargs)
|
||||
else:
|
||||
layer = ColumnParallelLinear(**layer_kwargs)
|
||||
assert isinstance(layer.quant_method, Mxfp8OnlineLinearMethod)
|
||||
|
||||
|
||||
def test_online_target_rejects_prequantized_layer(
|
||||
default_vllm_config, dist_init
|
||||
) -> None:
|
||||
"""A targets match participates in checkpoint compatibility checks."""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
prefix = "model.layers.0.self_attn.o_proj"
|
||||
quant_config = _fully_quantized_quark_config()
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(targets={prefix: "mxfp8"})
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="pre-quantized layer"):
|
||||
ColumnParallelLinear(
|
||||
input_size=32,
|
||||
output_size=32,
|
||||
bias=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
disable_tp=True,
|
||||
)
|
||||
|
||||
|
||||
def test_online_ignore_keeps_checkpoint_quantization_linear(
|
||||
default_vllm_config, dist_init, monkeypatch
|
||||
):
|
||||
"""Ignoring online quantization does not replace a checkpoint method (linear)."""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
quant_config = _fully_quantized_quark_config()
|
||||
prefix = "model.layers.0.self_attn.o_proj"
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(linear="mxfp8", ignore=[prefix])
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
quant_config.online_quantization_config,
|
||||
"get_quant_method",
|
||||
lambda *args: pytest.fail("online method should not be constructed"),
|
||||
)
|
||||
|
||||
layer = ColumnParallelLinear(
|
||||
input_size=32,
|
||||
output_size=32,
|
||||
bias=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
disable_tp=True,
|
||||
)
|
||||
|
||||
assert isinstance(layer.quant_method, QuarkLinearMethod)
|
||||
|
||||
|
||||
def test_online_quantization_rejects_prequantized_moe(
|
||||
default_vllm_config, dist_init
|
||||
) -> None:
|
||||
"""Online linear and MoE quantization reject a pre-quantized MoE layer."""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
prefix = "model.layers.0.mlp.experts"
|
||||
quant_config = _fully_quantized_quark_config()
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(linear="mxfp4", moe="mxfp4")
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="pre-quantized layer"):
|
||||
FusedMoEFactory(
|
||||
num_experts=4,
|
||||
top_k=2,
|
||||
hidden_size=32,
|
||||
intermediate_size=64,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
|
||||
def test_activation_only_override_applies_to_checkpoint_method(
|
||||
default_vllm_config, tmp_path
|
||||
) -> None:
|
||||
"""Activation-only overrides remain available to the checkpoint method."""
|
||||
_write_minimal_llama_config(tmp_path)
|
||||
model_config = ModelConfig(
|
||||
model=str(tmp_path),
|
||||
quantization="compressed-tensors",
|
||||
quantization_config=QuantizationConfigArgs(moe={"activation": "mxfp8"}),
|
||||
hf_overrides={
|
||||
"quantization_config": {
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": "pack-quantized",
|
||||
"config_groups": {},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
result = weight_utils.get_quant_config(model_config, LoadConfig())
|
||||
|
||||
assert isinstance(result, CompressedTensorsConfig)
|
||||
assert isinstance(result.online_quantization_config, OnlineQuantizationConfig)
|
||||
|
||||
default_vllm_config.model_config = model_config
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
_resolve_activation_key,
|
||||
)
|
||||
|
||||
assert _resolve_activation_key(None) == kMxfp8Dynamic
|
||||
|
||||
|
||||
def test_online_overlay_requires_normalized_quantization_config(tmp_path) -> None:
|
||||
"""The loader rejects a raw config dict instead of silently ignoring it."""
|
||||
_write_minimal_llama_config(tmp_path)
|
||||
model_config = ModelConfig(
|
||||
model=str(tmp_path),
|
||||
quantization="compressed-tensors",
|
||||
quantization_config={"linear": "mxfp8"},
|
||||
hf_overrides={
|
||||
"quantization_config": {
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": "pack-quantized",
|
||||
"config_groups": {},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
# assert isinstance(online_args, QuantizationConfigArgs)
|
||||
# in maybe_compose_online_quantization
|
||||
with pytest.raises(AssertionError):
|
||||
weight_utils.get_quant_config(model_config, LoadConfig())
|
||||
|
||||
|
||||
def test_online_overlay_loads_checkpoint_config_file(tmp_path) -> None:
|
||||
"""An online overlay must not bypass checkpoint config file loading."""
|
||||
_write_minimal_llama_config(tmp_path)
|
||||
(tmp_path / "quantize_config.json").write_text(
|
||||
'{"w_bit": 4, "q_group_size": 128, "zero_point": true}'
|
||||
)
|
||||
model_config = ModelConfig(
|
||||
model=str(tmp_path),
|
||||
quantization="awq",
|
||||
quantization_config=QuantizationConfigArgs(linear="mxfp8"),
|
||||
)
|
||||
|
||||
result = weight_utils.get_quant_config(model_config, LoadConfig())
|
||||
|
||||
assert isinstance(result, AutoAWQConfig)
|
||||
assert result.weight_bits == 4
|
||||
assert result.group_size == 128
|
||||
assert result.zero_point
|
||||
assert isinstance(result.online_quantization_config, OnlineQuantizationConfig)
|
||||
assert result.online_quantization_config.args is model_config.quantization_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"quantization,checkpoint_quantization_config,is_config_file",
|
||||
[
|
||||
pytest.param("fp8_per_block", None, False, id="fp8-per-block-online"),
|
||||
pytest.param("mxfp4", None, False, id="mxfp4-online"),
|
||||
pytest.param("mxfp8", None, False, id="mxfp8-online"),
|
||||
pytest.param(
|
||||
"mxfp8",
|
||||
{"quant_method": "mxfp8", "ignored_layers": []},
|
||||
False,
|
||||
id="mxfp8-inline-checkpoint-config",
|
||||
),
|
||||
pytest.param(
|
||||
"mxfp8",
|
||||
{"quant_method": "mxfp8", "ignored_layers": []},
|
||||
True,
|
||||
id="mxfp8-checkpoint-config-file",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_online_shorthand_selects_checkpoint_or_online_config(
|
||||
tmp_path, quantization, checkpoint_quantization_config, is_config_file
|
||||
) -> None:
|
||||
"""Online shorthands defer to checkpoint metadata when available."""
|
||||
|
||||
if is_config_file:
|
||||
(tmp_path / "hf_quant_config.json").write_text(
|
||||
'{"quant_method": "mxfp8", "ignored_layers": []}'
|
||||
)
|
||||
|
||||
_write_minimal_llama_config(
|
||||
tmp_path,
|
||||
None if is_config_file else checkpoint_quantization_config,
|
||||
)
|
||||
model_config = ModelConfig(model=str(tmp_path), quantization=quantization)
|
||||
|
||||
result = weight_utils.get_quant_config(model_config, LoadConfig())
|
||||
|
||||
if checkpoint_quantization_config is not None:
|
||||
assert isinstance(result, ModelOptMxFp8Config)
|
||||
assert result.online_quantization_config is None
|
||||
else:
|
||||
assert isinstance(result, OnlineQuantizationConfig)
|
||||
assert result.online_quantization_config is None
|
||||
|
||||
|
||||
def test_log_online_quantization_for_composable_config(monkeypatch) -> None:
|
||||
"""Composable configs log their nested online quantization results."""
|
||||
online_config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8"))
|
||||
online_config.quantized_layers = {
|
||||
"model.layers.0.self_attn.o_proj": ("linear", "mxfp8", None),
|
||||
"model.layers.1.self_attn.o_proj": ("linear", "mxfp8", None),
|
||||
}
|
||||
vllm_config = VllmConfig(quant_config=online_config)
|
||||
log_args = []
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.model_loader.base_loader.logger.info",
|
||||
lambda *args: log_args.append(args),
|
||||
)
|
||||
|
||||
log_online_quantization(vllm_config)
|
||||
|
||||
assert log_args == [
|
||||
(
|
||||
"Quantized %d layers of types: %s",
|
||||
2,
|
||||
"; ".join(online_config.quantized_layer_summaries),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_checkpoint_quantization_rejects_online_shorthand(tmp_path) -> None:
|
||||
"""A checkpoint quant method cannot be replaced by an online shorthand."""
|
||||
_write_minimal_llama_config(tmp_path, {"quant_method": "mxfp4"})
|
||||
|
||||
with pytest.raises(ValueError, match="does not match the quantization"):
|
||||
ModelConfig(model=str(tmp_path), quantization="fp8_per_channel")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls",
|
||||
("model_name,quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls"),
|
||||
[
|
||||
# simple case - quantization='fp8_per_tensor'
|
||||
(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_tensor",
|
||||
None,
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
@@ -148,12 +521,14 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
),
|
||||
# simple case - quantization='fp8_per_block'
|
||||
(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_block",
|
||||
None,
|
||||
Fp8PerBlockOnlineLinearMethod,
|
||||
Fp8PerBlockOnlineMoEMethod,
|
||||
),
|
||||
(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_channel",
|
||||
None,
|
||||
Fp8PtpcOnlineLinearMethod,
|
||||
@@ -161,6 +536,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
),
|
||||
# quantization='online' with per-layer-kind overrides
|
||||
(
|
||||
GRANITE_MODEL_NAME,
|
||||
"online",
|
||||
{
|
||||
"linear": "fp8_per_block",
|
||||
@@ -171,6 +547,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
),
|
||||
# quantization='online' with per-layer target patterns
|
||||
(
|
||||
GRANITE_MODEL_NAME,
|
||||
"online",
|
||||
{
|
||||
"targets": {
|
||||
@@ -183,6 +560,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
),
|
||||
# ignore with direct layer name
|
||||
(
|
||||
GRANITE_MODEL_NAME,
|
||||
"fp8_per_tensor",
|
||||
# qkv_proj is fused from q_proj/k_proj/v_proj. The shard regex
|
||||
# remains supported alongside direct fused-name regexes.
|
||||
@@ -191,11 +569,20 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
),
|
||||
(
|
||||
GRANITE_MODEL_NAME,
|
||||
"mxfp4",
|
||||
None,
|
||||
Mxfp4OnlineLinearMethod,
|
||||
Mxfp4OnlineMoEMethod,
|
||||
),
|
||||
pytest.param(
|
||||
PARTIALLY_PREQUANTIZED_MODEL_NAME,
|
||||
None,
|
||||
{"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}},
|
||||
Mxfp8OnlineLinearMethod,
|
||||
CompressedTensorsMoEMethod,
|
||||
id="partially_prequantized_checkpoint",
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"fp8_per_tensor",
|
||||
@@ -205,13 +592,15 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
"targets",
|
||||
"ignore",
|
||||
"mxfp4",
|
||||
"partially_prequantized_checkpoint",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
|
||||
)
|
||||
def test_online_quantization(
|
||||
quant_scheme: str,
|
||||
model_name: str,
|
||||
quant_scheme: str | None,
|
||||
online_quant_args: dict | None,
|
||||
expected_linear_cls,
|
||||
expected_moe_cls,
|
||||
@@ -243,38 +632,74 @@ def test_online_quantization(
|
||||
):
|
||||
pytest.skip(f"Skip test for online {quant_scheme} on XPU platform.")
|
||||
|
||||
model_name = "ibm-granite/granite-3.0-1b-a400m-base"
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
model_name,
|
||||
dtype="bfloat16",
|
||||
quantization=quant_scheme,
|
||||
model_config_kwargs={
|
||||
"quantization_config": resolve_quantization_config(
|
||||
quant_scheme, online_quant_args
|
||||
),
|
||||
"hf_overrides": {
|
||||
"num_hidden_layers": 3,
|
||||
"vocab_size": 256,
|
||||
"hidden_size": 256,
|
||||
"intermediate_size": 512,
|
||||
"num_attention_heads": 4,
|
||||
"num_key_value_heads": 4,
|
||||
"max_position_embeddings": 64,
|
||||
"num_local_experts": 4,
|
||||
"num_experts_per_tok": 2,
|
||||
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
model_name,
|
||||
model_config_kwargs={
|
||||
"quantization_config": resolve_quantization_config(
|
||||
None, online_quant_args
|
||||
)
|
||||
},
|
||||
},
|
||||
model_loader_cls=DummyModelLoader,
|
||||
)
|
||||
model_loader_cls=DummyModelLoader,
|
||||
)
|
||||
assert isinstance(
|
||||
model.model.layers[1].self_attn.o_proj.quant_method,
|
||||
expected_linear_cls,
|
||||
)
|
||||
assert isinstance(
|
||||
model.model.layers[0].mlp.experts._quant_method,
|
||||
expected_moe_cls,
|
||||
)
|
||||
assert isinstance(
|
||||
model.model.layers[0].self_attn.o_proj.quant_method,
|
||||
CompressedTensorsLinearMethod,
|
||||
)
|
||||
else:
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
model_name,
|
||||
dtype="bfloat16",
|
||||
quantization=quant_scheme,
|
||||
model_config_kwargs={
|
||||
"quantization_config": resolve_quantization_config(
|
||||
quant_scheme, online_quant_args
|
||||
),
|
||||
"hf_overrides": {
|
||||
"num_hidden_layers": 3,
|
||||
"vocab_size": 256,
|
||||
"hidden_size": 256,
|
||||
"intermediate_size": 512,
|
||||
"num_attention_heads": 4,
|
||||
"num_key_value_heads": 4,
|
||||
"max_position_embeddings": 64,
|
||||
"num_local_experts": 4,
|
||||
"num_experts_per_tok": 2,
|
||||
},
|
||||
},
|
||||
model_loader_cls=DummyModelLoader,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(Attention, "forward", lambda _, q, k, v: q.contiguous())
|
||||
|
||||
o_proj = model.model.layers[0].self_attn.o_proj
|
||||
moe = model.model.layers[0].block_sparse_moe.experts
|
||||
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
|
||||
o_proj = model.model.layers[1].self_attn.o_proj
|
||||
moe = model.model.layers[0].mlp.experts
|
||||
else:
|
||||
o_proj = model.model.layers[0].self_attn.o_proj
|
||||
moe = model.model.layers[0].block_sparse_moe.experts
|
||||
assert isinstance(o_proj.quant_method, expected_linear_cls)
|
||||
assert isinstance(moe._quant_method, expected_moe_cls)
|
||||
|
||||
if quant_scheme == "mxfp4":
|
||||
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance(
|
||||
o_proj.quant_method.kernel, MarlinMxfp8LinearKernel
|
||||
):
|
||||
assert o_proj.weight.dtype == torch.int32
|
||||
elif model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance(
|
||||
o_proj.quant_method.kernel, EmulationMxfp8LinearKernel
|
||||
):
|
||||
assert o_proj.weight.dtype == torch.bfloat16
|
||||
elif model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
|
||||
assert o_proj.weight.dtype == MXFP8_VALUE_DTYPE
|
||||
elif quant_scheme == "mxfp4":
|
||||
assert o_proj.weight.dtype == torch.uint8
|
||||
elif current_platform.is_cuda() or current_platform.is_xpu():
|
||||
assert o_proj.weight.dtype == torch.float8_e4m3fn
|
||||
@@ -301,7 +726,11 @@ def test_online_quantization(
|
||||
layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod
|
||||
)
|
||||
|
||||
if isinstance(online_quant_args, dict) and "targets" in online_quant_args:
|
||||
if (
|
||||
model_name == GRANITE_MODEL_NAME
|
||||
and isinstance(online_quant_args, dict)
|
||||
and "targets" in online_quant_args
|
||||
):
|
||||
for layer in model.model.layers:
|
||||
assert isinstance(
|
||||
layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod
|
||||
@@ -319,8 +748,13 @@ def test_online_quantization(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_online_quantization_loads_real_weights(vllm_runner, monkeypatch) -> None:
|
||||
"""Verify online quantization loads a Granite-MoE checkpoint end to end."""
|
||||
@pytest.mark.parametrize(
|
||||
"model_name", [GRANITE_MODEL_NAME, PARTIALLY_PREQUANTIZED_MODEL_NAME]
|
||||
)
|
||||
def test_online_quantization_loads_real_weights(
|
||||
model_name, vllm_runner, monkeypatch
|
||||
) -> None:
|
||||
"""Verify online quantization loads real checkpoints end to end."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
@@ -336,26 +770,52 @@ def test_online_quantization_loads_real_weights(vllm_runner, monkeypatch) -> Non
|
||||
|
||||
monkeypatch.setattr(GraniteMoeModel, "load_weights", load_weights)
|
||||
|
||||
with vllm_runner(
|
||||
"ibm-granite/granite-3.0-1b-a400m-base",
|
||||
quantization="fp8_per_tensor",
|
||||
dtype="bfloat16",
|
||||
enforce_eager=True,
|
||||
hf_overrides={"num_hidden_layers": 3},
|
||||
max_model_len=16,
|
||||
max_num_seqs=1,
|
||||
) as llm:
|
||||
if model_name == GRANITE_MODEL_NAME:
|
||||
runner_kwargs = {
|
||||
"quantization": "fp8_per_tensor",
|
||||
"dtype": "bfloat16",
|
||||
"enforce_eager": True,
|
||||
"hf_overrides": {"num_hidden_layers": 3},
|
||||
"max_model_len": 16,
|
||||
"max_num_seqs": 1,
|
||||
}
|
||||
else:
|
||||
runner_kwargs = {
|
||||
"quantization_config": {
|
||||
"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}
|
||||
},
|
||||
"enforce_eager": True,
|
||||
}
|
||||
|
||||
def check_model(model):
|
||||
layer = model.model.layers[0]
|
||||
assert isinstance(
|
||||
layer.self_attn.o_proj.quant_method,
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
layer.block_sparse_moe.experts._quant_method,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
)
|
||||
with vllm_runner(model_name, **runner_kwargs) as llm:
|
||||
if model_name == GRANITE_MODEL_NAME:
|
||||
|
||||
def check_model(model):
|
||||
layer = model.model.layers[0]
|
||||
assert isinstance(
|
||||
layer.self_attn.o_proj.quant_method,
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
layer.block_sparse_moe.experts._quant_method,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
def check_model(model):
|
||||
assert isinstance(
|
||||
model.model.layers[1].self_attn.o_proj.quant_method,
|
||||
Mxfp8OnlineLinearMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
model.model.layers[0].self_attn.o_proj.quant_method,
|
||||
CompressedTensorsLinearMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
model.model.layers[0].mlp.experts._quant_method,
|
||||
CompressedTensorsMoEMethod,
|
||||
)
|
||||
|
||||
llm.apply_model(check_model)
|
||||
outputs = llm.generate_greedy(["Hello my name is"], max_tokens=1)
|
||||
@@ -457,8 +917,8 @@ def test_online_quantization_targets_ignore_collision() -> None:
|
||||
)
|
||||
)
|
||||
with pytest.raises(ValueError, match="matches both quantization_config.ignore"):
|
||||
config._dispatch_target(
|
||||
"model.layers.0.self_attn.o_proj", Mock(spec=LinearBase)
|
||||
config.resolve_quant_method_cls(
|
||||
Mock(spec=LinearBase), "model.layers.0.self_attn.o_proj"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -354,7 +354,7 @@ def test_online_mxfp4_moe_matches_quark(
|
||||
weight_quant_key=kMxfp4Static,
|
||||
activation_quant_key=kMxfp4Dynamic,
|
||||
)
|
||||
online_method = Mxfp4OnlineMoEMethod(layer=online_layer)
|
||||
online_method = Mxfp4OnlineMoEMethod(moe=online_layer.moe_config)
|
||||
|
||||
# `RoutedExperts.__init__` applies this round-up in production; these
|
||||
# layers are built without a quant config, so it is applied explicitly.
|
||||
|
||||
@@ -95,6 +95,12 @@ def test_resolve_shorthand_only_populates_both_slots():
|
||||
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("quantization", ["mxfp4", "mxfp8"])
|
||||
def test_resolve_colliding_shorthand_is_deferred(quantization: str):
|
||||
"""Checkpoint metadata determines whether an MXFP shorthand is online."""
|
||||
assert resolve_quantization_config(quantization, None) is None
|
||||
|
||||
|
||||
def test_resolve_int8_shorthand_leaves_linear_unset():
|
||||
# int8_per_channel_weight_only is MoE-only; linear stays None so that
|
||||
# OnlineQuantizationConfig leaves Linear layers in full precision.
|
||||
@@ -122,12 +128,9 @@ def test_resolve_merges_explicit_over_shorthand():
|
||||
assert args.moe == QuantSpec(weight=kFp8StaticTensorSym)
|
||||
|
||||
|
||||
def test_resolve_rejects_quantization_config_with_non_shorthand_quant():
|
||||
# If --quantization names something other than an online shorthand,
|
||||
# quantization_config is not allowed via this path (checkpoint quant
|
||||
# paths read it directly off ModelConfig instead).
|
||||
with pytest.raises(ValueError, match="quantization_config is only supported"):
|
||||
resolve_quantization_config("gptq", {"linear": "fp8_per_block"})
|
||||
def test_resolve_quantization_config_with_checkpoint_quantization():
|
||||
args = resolve_quantization_config("gptq", {"linear": "fp8_per_block"})
|
||||
assert args == QuantizationConfigArgs(linear="fp8_per_block")
|
||||
|
||||
|
||||
# ---- QUANT_KEY_NAMES coverage --------------------------------------------
|
||||
@@ -206,6 +209,6 @@ def test_targets_reject_moe_only_shorthand_for_linear_layer():
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not define a QuantSpec"):
|
||||
config._dispatch_target(
|
||||
"model.layers.0.self_attn.o_proj", Mock(spec=LinearBase)
|
||||
config.resolve_quant_method_cls(
|
||||
Mock(spec=LinearBase), "model.layers.0.self_attn.o_proj"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user