[Quantization] Enable shared expert fusion compatibility with online shared_expert quantization (showcase: along Quark MXFP4 routed experts) (#55686)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: fxmarty-amd <felmarty@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-10-08 12:04:46 -04:00
committed by GitHub
co-authored by OpenAI Codex Misha Goin
parent e2414ec1db
commit 9f3d70e081
24 changed files with 984 additions and 73 deletions
+1
View File
@@ -162,6 +162,7 @@ def test_routed_experts_loads_per_expert_biases():
class Loader:
quant_config = None
quant_method = object()
_fused_shared_expert_quantizer = None
moe_config = SimpleNamespace(
is_act_and_mul=True,
tp_rank=0,
+2 -3
View File
@@ -2,11 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests quantization configuration matching utilities."""
from unittest.mock import Mock
import pytest
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
should_ignore_layer,
)
@@ -58,7 +57,7 @@ def test_online_ignore_supports_fnmatch_patterns():
)
with pytest.raises(ValueError, match="matches both quantization_config.ignore"):
config.resolve_quant_method_cls(Mock(), layer_name)
config.resolve_quant_method_cls(LinearBase, layer_name)
@pytest.mark.parametrize(
+3 -6
View File
@@ -195,7 +195,6 @@ def test_qianfan_online_fp8_keeps_vision_layers_unquantized(
config,
)
layer = Mock(spec=LinearBase)
for prefix in (
"vision_model.encoder.layers.0.attn.qkv",
"vision_model.encoder.layers.1.mlp.fc2",
@@ -203,10 +202,10 @@ def test_qianfan_online_fp8_keeps_vision_layers_unquantized(
"mlp1.1",
"mlp1.3",
):
assert config.resolve_quant_method_cls(layer, prefix) is None
assert config.resolve_quant_method_cls(LinearBase, prefix) is None
resolved = config.resolve_quant_method_cls(
layer, "language_model.model.layers.0.self_attn.qkv_proj"
LinearBase, "language_model.model.layers.0.self_attn.qkv_proj"
)
assert resolved is not None and resolved[-1] is method_cls
assert other_config.ignored_layers == []
@@ -1320,9 +1319,7 @@ def test_online_quantization_targets_ignore_collision() -> None:
)
)
with pytest.raises(ValueError, match="matches both quantization_config.ignore"):
config.resolve_quant_method_cls(
Mock(spec=LinearBase), "model.layers.0.self_attn.o_proj"
)
config.resolve_quant_method_cls(LinearBase, "model.layers.0.self_attn.o_proj")
def test_online_quantization_targets_reject_unsupported_layer() -> None:
@@ -0,0 +1,637 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests ROCm AITER fused shared experts with online quantization."""
import json
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import pytest
import torch
from tests.quantization.utils import load_model_without_vllm_runner
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config.model import ModelConfig
from vllm.config.quantization import (
QuantizationConfigArgs,
resolve_quantization_config,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.experts import rocm_aiter_moe
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.quark.quark_moe import (
QuarkOCP_MX_MoEMethod,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
mxfp4_quantize,
)
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
from vllm.model_executor.model_loader.reload.layerwise import (
finalize_layerwise_reload,
initialize_layerwise_reload,
record_metadata_for_reloading,
)
from vllm.model_executor.model_loader.utils import get_model_architecture
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm AITER shared-expert tests require ROCm.",
)
_QUARK_MXFP4_CONFIG: dict[str, Any] = {
"global_quant_config": {
"input_tensors": {
"dtype": "fp4",
"is_dynamic": True,
"qscheme": "per_group",
"ch_axis": -1,
"group_size": 32,
"block_size": None,
"symmetric": None,
"round_method": "half_even",
"scale_type": "float",
"scale_format": "e8m0",
"scale_calculation_mode": "even",
"mx_element_dtype": None,
"observer_cls": "PerBlockMXObserver",
"is_scale_quant": False,
"enable_buffer_reuse": False,
"max_input_numel": 4194304,
},
"output_tensors": None,
"weight": {
"dtype": "fp4",
"is_dynamic": False,
"qscheme": "per_group",
"ch_axis": -1,
"group_size": 32,
"block_size": None,
"symmetric": None,
"round_method": "half_even",
"scale_type": "float",
"scale_format": "e8m0",
"scale_calculation_mode": "even",
"mx_element_dtype": None,
"observer_cls": "PerBlockMXObserver",
"is_scale_quant": False,
"enable_buffer_reuse": False,
"max_input_numel": 4194304,
},
"bias": None,
"target_device": None,
},
"algo_config": None,
"softmax_quant_spec": None,
"quant_method": "quark",
"layer_type_quant_config": {},
"layer_quant_config": {},
"exclude": ["re:^(?!.*\\.mlp\\.experts(?:\\.|$)).*$"],
"kv_cache_quant_config": {},
"kv_cache_post_rope": False,
"quant_mode": "eager_mode",
"version": "0.12+9d3d471cdf1",
"export": {
"kv_cache_group": [],
"min_kv_scale": 0.0,
"pack_method": "reorder",
"weight_format": "real_quantized",
"weight_merge_groups": None,
},
}
_QUARK_MXFP4_LAYER_CONFIG: dict[str, Any] = _QUARK_MXFP4_CONFIG["global_quant_config"]
_QUARK_MXFP4_CONFIG = {
**_QUARK_MXFP4_CONFIG,
"global_quant_config": {
"input_tensors": {},
"output_tensors": None,
"weight": {},
"bias": None,
"target_device": None,
},
"layer_quant_config": {"*mlp.experts*": _QUARK_MXFP4_LAYER_CONFIG},
}
class _StubAttention(torch.nn.Module):
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__()
self.o_proj = SimpleNamespace(reduce_results=True)
@pytest.fixture(autouse=True)
def reset_aiter_shared_expert_topk_metadata(monkeypatch: pytest.MonkeyPatch):
"""Isolate AITER's process-global shared-expert routing buffer."""
monkeypatch.setattr(rocm_aiter_moe, "aiter_topK_meta_data", None)
rocm_aiter_moe.init_aiter_topK_meta_data.cache_clear()
yield
rocm_aiter_moe.init_aiter_topK_meta_data.cache_clear()
def _write_minimal_moe_config(model_path: Path, architecture: str) -> None:
"""Write a compact MoE config with one shared expert."""
config = {
"architectures": [architecture],
"hidden_size": 256,
"intermediate_size": 512,
"moe_intermediate_size": 256,
"num_attention_heads": 4,
"num_key_value_heads": 4,
"head_dim": 64,
"num_hidden_layers": 1,
"vocab_size": 256,
"max_position_embeddings": 64,
"q_lora_rank": 64,
"kv_lora_rank": 512,
"qk_nope_head_dim": 128,
"qk_rope_head_dim": 64,
"v_head_dim": 128,
"n_routed_experts": 4,
"num_experts": 4,
"num_experts_per_tok": 1,
"n_group": 1,
"topk_group": 1,
"n_shared_experts": 1,
"first_k_dense_replace": 0,
"moe_layer_freq": 1,
"rms_norm_eps": 1e-6,
"hidden_act": "silu",
"tie_word_embeddings": False,
"rope_theta": 10000,
"quantization_config": _QUARK_MXFP4_CONFIG,
}
if architecture == "AXK1ForCausalLM":
config["model_type"] = "axk1"
elif architecture == "MiniMaxM3SparseForCausalLM":
text_config = {
**config,
"model_type": "minimax_m3_text",
"dense_intermediate_size": 512,
"hidden_act": "swigluoai",
"num_local_experts": 4,
"n_shared_experts": 1,
"moe_layer_freq": [1],
"sparse_attention_config": None,
"quantization_config": {
**_QUARK_MXFP4_CONFIG,
"layer_quant_config": {
"*block_sparse_moe.experts*": _QUARK_MXFP4_LAYER_CONFIG
},
"exclude": [r"re:^(?!.*\.block_sparse_moe\.experts(?:\.|$)).*$"],
},
}
config = {
"architectures": [architecture],
"model_type": "minimax_m3_vl",
"text_config": text_config,
}
elif architecture == "Glm4MoeForCausalLM":
config["model_type"] = "glm4_moe"
elif architecture == "Glm4MoeLiteForCausalLM":
config["model_type"] = "glm4_moe_lite"
elif architecture in ("Qwen3NextForCausalLM", "Qwen3_5MoeForCausalLM"):
config.update(
model_type=(
"qwen3_next"
if architecture == "Qwen3NextForCausalLM"
else "qwen3_5_moe_text"
),
layer_types=[
"linear_attention"
if architecture == "Qwen3NextForCausalLM"
else "full_attention"
],
head_dim=64,
linear_key_head_dim=64,
linear_value_head_dim=64,
linear_num_key_heads=4,
linear_num_value_heads=4,
shared_expert_intermediate_size=256,
)
elif architecture == "Qwen4ExpForCausalLM":
config.update(
model_type="qwen4_exp_text",
layer_types=["full_attention"],
head_dim=64,
linear_key_head_dim=64,
linear_value_head_dim=64,
linear_num_key_heads=4,
linear_num_value_heads=4,
shared_expert_intermediate_size=256,
eos_token_id=1,
hc_count=2,
hc_lowrank=4,
ple_layer_ids=[],
)
elif architecture == "Glm5NextForCausalLM":
config.update(
model_type="glm5_next",
layer_types=["linear_attention"],
mlp_layer_types=["sparse"],
qk_rope_head_dim=0,
index_topk=1,
index_kpool=1,
pad_token_id=None,
scoring_func="sigmoid",
topk_method="noaux_tc",
mhc=False,
)
else:
config["model_type"] = (
"deepseek_v3"
if "V3" in architecture
else "deepseek_v32"
if architecture == "DeepseekV32ForCausalLM"
else "deepseek_v2"
)
if architecture in ("DeepseekV32ForCausalLM", "GlmMoeDsaForCausalLM"):
config.update(
index_topk=1,
index_kpool=1,
index_n_heads=1,
index_head_dim=32,
index_kv_lora_rank=32,
)
(model_path / "config.json").write_text(json.dumps(config))
@pytest.mark.parametrize(
"load_path", ["load_weights", "fused_gate_up", "weight_loader"]
)
def test_online_shared_expert_loads_bf16_weights_into_mxfp4_slot(
default_vllm_config,
dist_init,
load_path: str,
) -> None:
"""A BF16 shared expert is quantized while routed MXFP4 weights are loaded.
`weight_loader` covers DeepSeek-style model loaders, which bypass
`RoutedExperts.load_weights`.
"""
default_vllm_config.model_config = ModelConfig()
hidden_size = intermediate_size = 64
num_routed_experts = 2
device = current_platform.device_type
online_config = OnlineQuantizationConfig(
QuantizationConfigArgs(targets={"*shared_expert*": "mxfp4"})
)
quant_config = QuarkConfig(_QUARK_MXFP4_CONFIG)
quant_config.online_quantization_config = online_config
with torch.device(device):
runner = FusedMoEFactory(
num_experts=num_routed_experts,
top_k=1,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
n_shared_experts=1,
fuse_shared_experts=True,
shared_expert_prefix="model.layers.0.mlp.shared_expert",
prefix="model.layers.0.mlp.experts",
quant_config=quant_config,
)
layer = runner.routed_experts
assert isinstance(layer.quant_method, QuarkOCP_MX_MoEMethod)
routed_gate, _ = mxfp4_quantize(
torch.randn(intermediate_size, hidden_size, dtype=torch.bfloat16)
)
routed_up, _ = mxfp4_quantize(
torch.randn(intermediate_size, hidden_size, dtype=torch.bfloat16)
)
routed_down, _ = mxfp4_quantize(
torch.randn(hidden_size, intermediate_size, dtype=torch.bfloat16)
)
shared_gate = torch.randn(intermediate_size, hidden_size, dtype=torch.bfloat16)
shared_up = torch.randn(intermediate_size, hidden_size, dtype=torch.bfloat16)
shared_down = torch.randn(hidden_size, intermediate_size, dtype=torch.bfloat16)
weights = [
("0.gate_proj.weight", routed_gate),
("0.up_proj.weight", routed_up),
("0.down_proj.weight", routed_down),
]
if load_path == "load_weights":
weights += [
("2.gate_proj.weight", shared_gate),
("2.up_proj.weight", shared_up),
("2.down_proj.weight", shared_down),
]
elif load_path == "fused_gate_up":
weights += [
("2.gate_up_proj.weight", torch.cat([shared_gate, shared_up])),
("2.down_proj.weight", shared_down),
]
list(layer.load_weights(weights))
if load_path == "weight_loader":
for stem, shard_id, weight in (
("w13", "w1", shared_gate),
("w13", "w3", shared_up),
("w2", "w2", shared_down),
):
param = getattr(layer, f"{stem}_weight")
param.weight_loader(
param, weight, f"{layer.layer_name}.{stem}_weight", shard_id, 2
)
expected_gate, expected_gate_scale = mxfp4_quantize(shared_gate)
expected_up, expected_up_scale = mxfp4_quantize(shared_up)
expected_down, expected_down_scale = mxfp4_quantize(shared_down)
assert torch.equal(layer.w13_weight[2, :intermediate_size], expected_gate)
assert torch.equal(layer.w13_weight[2, intermediate_size:], expected_up)
assert torch.equal(layer.w2_weight[2], expected_down)
assert torch.equal(
layer.w13_weight_scale[2, :intermediate_size], expected_gate_scale
)
assert torch.equal(layer.w13_weight_scale[2, intermediate_size:], expected_up_scale)
assert torch.equal(layer.w2_weight_scale[2], expected_down_scale)
def test_online_shared_expert_reload_compatibility(
default_vllm_config,
dist_init,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Reload a BF16 shared weight into its MXFP4 slot via meta staging."""
default_vllm_config.model_config = ModelConfig()
online_config = OnlineQuantizationConfig(
QuantizationConfigArgs(targets={"*shared_expert*": "mxfp4"})
)
quant_config = QuarkConfig(_QUARK_MXFP4_CONFIG)
quant_config.online_quantization_config = online_config
with torch.device(current_platform.device_type):
layer = FusedMoEFactory(
num_experts=2,
top_k=1,
hidden_size=64,
intermediate_size=64,
n_shared_experts=1,
fuse_shared_experts=True,
shared_expert_prefix="model.layers.0.mlp.shared_expert",
prefix="model.layers.0.mlp.experts",
quant_config=quant_config,
).routed_experts
monkeypatch.setattr(
layer.quant_method, "process_weights_after_loading", lambda _: None
)
routed_weights = []
for expert_id in range(2):
for projection in ("gate_proj", "up_proj", "down_proj"):
weight, scale = mxfp4_quantize(
torch.randn(64, 64, dtype=torch.bfloat16)
)
prefix = f"{expert_id}.{projection}"
routed_weights.extend(
[(f"{prefix}.weight", weight), (f"{prefix}.weight_scale", scale)]
)
shared_gate = torch.randn(64, 64, dtype=torch.bfloat16)
shared_up = torch.randn(64, 64, dtype=torch.bfloat16)
shared_down = torch.randn(64, 64, dtype=torch.bfloat16)
def checkpoint_weights(gate: torch.Tensor):
return [
*routed_weights,
("2.gate_proj.weight", gate),
("2.up_proj.weight", shared_up),
("2.down_proj.weight", shared_down),
]
list(layer.load_weights(checkpoint_weights(shared_gate)))
old_weight = layer.w13_weight[2, :64].clone()
reloaded_gate = shared_gate + 1
expected_weight, expected_scale = mxfp4_quantize(reloaded_gate)
assert not torch.equal(old_weight, expected_weight)
record_metadata_for_reloading(layer)
initialize_layerwise_reload(layer)
assert layer.w13_weight.is_meta
list(layer.load_weights(checkpoint_weights(reloaded_gate)))
finalize_layerwise_reload(layer, default_vllm_config.model_config)
assert not layer.w13_weight.is_meta
assert torch.equal(layer.w13_weight[2, :64], expected_weight)
assert torch.equal(layer.w13_weight_scale[2, :64], expected_scale)
# TODO: Add DeepseekV4ForConditionalGeneration once it uses
# resolve_layer_fused_shared_expert.
@pytest.mark.parametrize(
"architecture",
[
"AXK1ForCausalLM",
"DeepseekForCausalLM",
"DeepseekV2ForCausalLM",
"DeepseekV32ForCausalLM",
"DeepseekV3ForCausalLM",
"Glm4MoeForCausalLM",
"Glm4MoeLiteForCausalLM",
"Glm5NextForCausalLM",
"GlmMoeDsaForCausalLM",
"MiniMaxM3SparseForCausalLM",
"Qwen3NextForCausalLM",
"Qwen3_5MoeForCausalLM",
"Qwen4ExpForCausalLM",
],
)
def test_online_quantization(
architecture: str,
tmp_path: Path,
monkeypatch,
dist_init,
workspace_init,
) -> None:
"""Online MXFP4 fuses each architecture's shared expert into its MoE."""
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", "1")
rocm_aiter_ops.refresh_env_variables()
if architecture == "Glm5NextForCausalLM":
from vllm.models.glm5next.common import model as glm5_next_model
for attention in ("Glm5NextLinearAttention", "Glm5NextMLAAttention"):
monkeypatch.setattr(glm5_next_model, attention, _StubAttention)
monkeypatch.setattr(
glm5_next_model, "_fused_shared_experts_tuned", lambda _: True
)
elif architecture == "Glm4MoeLiteForCausalLM":
from vllm.model_executor.models import glm4_moe_lite
monkeypatch.setattr(glm4_moe_lite, "Glm4MoeLiteAttention", _StubAttention)
monkeypatch.setattr(glm4_moe_lite, "Glm4MoeLiteMLAAttention", _StubAttention)
elif architecture == "DeepseekV32ForCausalLM":
from vllm.models.deepseek_v32.amd import model as deepseek_v32_model
monkeypatch.setattr(
deepseek_v32_model, "DeepseekV32MLAAttention", _StubAttention
)
elif architecture == "Qwen3_5MoeForCausalLM":
from vllm.model_executor.models import qwen3_5
monkeypatch.setattr(qwen3_5, "Qwen3NextAttention", _StubAttention)
elif architecture == "Qwen4ExpForCausalLM":
from vllm.models.qwen4_exp.amd import model as qwen4_exp_model
monkeypatch.setattr(qwen4_exp_model, "Qwen3NextAttention", _StubAttention)
_write_minimal_moe_config(tmp_path, architecture)
expected_shared_expert_name = (
"shared_expert" if architecture.startswith("Qwen") else "shared_experts"
)
target_pattern = f"*{expected_shared_expert_name}*"
logged_messages: list[str] = []
logged_warnings: list[str] = []
def record_info(message: str, *args: object, **_kwargs: object) -> None:
if "Quantizing " in message and "of types" in message:
logged_messages.append(message % args)
def record_warning(message: str, *args: object, **_kwargs: object) -> None:
logged_warnings.append(message % args)
monkeypatch.setattr(
"vllm.model_executor.model_loader.base_loader.logger.info", record_info
)
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.utils.logger.warning", record_warning
)
model, vllm_config = load_model_without_vllm_runner(
str(tmp_path),
dtype="bfloat16",
quantization="quark",
model_config_kwargs={
"quantization_config": resolve_quantization_config(
"quark", {"targets": {target_pattern: "mxfp4"}}
)
},
model_loader_cls=DummyModelLoader,
)
expected_model_cls, resolved_architecture = get_model_architecture(
vllm_config.model_config
)
assert resolved_architecture == architecture
assert isinstance(model, expected_model_cls)
fused_moes = [
module
for module in model.modules()
if getattr(module, "is_fused_shared_expert_enabled", False)
and hasattr(module, "experts")
]
assert len(fused_moes) == 1, f"{architecture} did not enable FSE"
moe = fused_moes[0]
shared_expert_name = (
"shared_expert" if hasattr(moe, "shared_expert") else "shared_experts"
)
assert shared_expert_name == expected_shared_expert_name
assert getattr(moe, shared_expert_name) is None
routed_experts = moe.experts.routed_experts
assert isinstance(routed_experts.quant_method, QuarkOCP_MX_MoEMethod)
shared_expert_prefix = routed_experts.moe_config.shared_expert_prefix
assert shared_expert_prefix is not None
assert shared_expert_prefix.endswith(shared_expert_name)
assert routed_experts._fused_shared_expert_quantizer is not None
expert_map_manager = routed_experts.expert_map_manager
assert expert_map_manager.num_fused_shared_experts == 1
assert expert_map_manager.map_global_to_local(
routed_experts.moe_config.num_experts
) == (routed_experts.moe_config.num_experts)
expected_weight_dtype = {
Mxfp4MoeBackend.AITER_MXFP4_BF16: torch.float4_e2m1fn_x2,
Mxfp4MoeBackend.AITER_MXFP4_MXFP4: torch.float4_e2m1fn_x2,
Mxfp4MoeBackend.EMULATION: torch.uint8,
}[routed_experts.quant_method.mxfp4_backend]
assert routed_experts.w13_weight.dtype == expected_weight_dtype
assert routed_experts.w13_weight_scale.dtype == torch.uint8
assert routed_experts.w2_weight.dtype == expected_weight_dtype
assert routed_experts.w2_weight_scale.dtype == torch.uint8
assert vllm_config.quant_config is not None
assert vllm_config.quant_config.online_quantization_config is not None
assert not any(
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
"cannot be enabled" in warning
for warning in logged_warnings
)
if architecture == "MiniMaxM3SparseForCausalLM":
# TODO: Use resolve_layer_fused_shared_expert in MiniMax M3 and remove
# this branch once its virtual fused projections are registered.
assert logged_messages == ["Quantizing 0 layers of types: "]
else:
assert logged_messages == [
"Quantizing 2 layers of types: "
f"mlp.{shared_expert_name}.down_proj: 1 "
f"(from targets: {target_pattern}, mxfp4); "
f"mlp.{shared_expert_name}.gate_up_proj: 1 "
f"(from targets: {target_pattern}, mxfp4)"
]
def test_online_gate_up_proj_target_disables_fse(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
dist_init,
workspace_init,
) -> None:
"""A partial shared-expert target quantizes while disabling FSE."""
architecture = "DeepseekForCausalLM"
target_pattern = "*shared_experts.gate_up_proj*"
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", "1")
rocm_aiter_ops.refresh_env_variables()
_write_minimal_moe_config(tmp_path, architecture)
logged_messages: list[str] = []
logged_warnings: list[str] = []
def record_info(message: str, *args: object, **_kwargs: object) -> None:
if "Quantizing " in message and "of types" in message:
logged_messages.append(message % args)
def record_warning(message: str, *args: object, **_kwargs: object) -> None:
logged_warnings.append(message % args)
monkeypatch.setattr(
"vllm.model_executor.model_loader.base_loader.logger.info", record_info
)
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.utils.logger.warning", record_warning
)
model, _ = load_model_without_vllm_runner(
str(tmp_path),
dtype="bfloat16",
quantization="quark",
model_config_kwargs={
"quantization_config": resolve_quantization_config(
"quark", {"targets": {target_pattern: "mxfp4"}}
)
},
model_loader_cls=DummyModelLoader,
)
moe = next(
module
for module in model.modules()
if hasattr(module, "is_fused_shared_expert_enabled")
and hasattr(module, "shared_experts")
)
assert not moe.is_fused_shared_expert_enabled
assert moe.shared_experts is not None
assert logged_messages == [
"Quantizing 1 layers of types: "
"mlp.shared_experts.gate_up_proj: 1 "
f"(from targets: {target_pattern}, mxfp4)"
]
assert logged_warnings == [
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but cannot be "
"enabled - skipping for this layer: online quantization targets only "
"part of the shared expert at model.layers.0.mlp.shared_experts."
]
@@ -2,8 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for QuantizationConfigArgs parsing."""
from unittest.mock import Mock
import pytest
from tests.quantization.utils import quant_config_args, quant_spec
@@ -244,6 +242,4 @@ def test_targets_reject_moe_only_shorthand_for_linear_layer():
)
with pytest.raises(ValueError, match="does not define a QuantSpec"):
config.resolve_quant_method_cls(
Mock(spec=LinearBase), "model.layers.0.self_attn.o_proj"
)
config.resolve_quant_method_cls(LinearBase, "model.layers.0.self_attn.o_proj")
+23 -6
View File
@@ -15,7 +15,11 @@ from vllm.config import (
VllmConfig,
set_current_vllm_config,
)
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
from vllm.config.quantization import (
QuantizationConfigArgs,
QuantSpec,
resolve_quantization_config,
)
from vllm.model_executor.layers.quantization import get_quantization_config
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.platforms import current_platform
@@ -40,10 +44,17 @@ def _limit_num_hidden_layers(
original_load_weights = model.load_weights
def should_load_weight(name: str) -> bool:
for prefix in ("model.layers.", "layers."):
if name.startswith(prefix):
layer_idx = int(name.removeprefix(prefix).split(".", 1)[0])
return layer_idx < num_hidden_layers
layers_prefix = "layers."
if name.startswith(layers_prefix):
layer_name = name.removeprefix(layers_prefix)
elif ".layers." in name:
layer_name = name.split(".layers.", maxsplit=1)[1]
else:
return True
layer_idx_str = layer_name.split(".", maxsplit=1)[0]
if layer_idx_str.isdigit():
return int(layer_idx_str) < num_hidden_layers
return True
def load_weights(weights):
@@ -71,13 +82,19 @@ def load_model_without_vllm_runner(
quantization=quantization,
**(model_config_kwargs or {}),
)
model_config.quantization_config = resolve_quantization_config(
model_config.quantization, model_config.quantization_config
)
vllm_config_args = dict(vllm_config_kwargs or {})
vllm_config_args.setdefault(
"compilation_config", CompilationConfig(mode=CompilationMode.NONE)
)
vllm_config = VllmConfig(model_config=model_config, **vllm_config_args)
hf_overrides = (model_config_kwargs or {}).get("hf_overrides") or {}
num_hidden_layers = hf_overrides.get("num_hidden_layers")
text_config_overrides = hf_overrides.get("text_config") or {}
num_hidden_layers = text_config_overrides.get(
"num_hidden_layers", hf_overrides.get("num_hidden_layers")
)
with set_current_vllm_config(vllm_config):
model_loader = model_loader_cls(vllm_config.load_config)