[MOE] Standardize and abstract fused shared expert optimization selection (#51695)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-08-18 12:56:56 -04:00
committed by GitHub
parent f9f066d195
commit 88b2bff2c6
20 changed files with 1288 additions and 281 deletions
@@ -0,0 +1,700 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib
from copy import deepcopy
from types import SimpleNamespace
from typing import Any, cast
from unittest.mock import patch
import pytest
import torch
from torch import nn
import vllm.config as vllm_config_module
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import utils as fused_moe_utils
from vllm.model_executor.layers.fused_moe.layer import determine_expert_counts
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.config_utils import (
is_shared_expert_quant_fse_compatible,
)
from vllm.model_executor.models.utils import PPMissingLayer
from vllm.models.deepseek_v4 import quant_config as deepseek_v4_quant_config
from vllm.models.minimax_m3.amd import model as minimax_m3_model
from vllm.transformers_utils.configs.minimax_m3 import MiniMaxM3TextConfig
from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
_QUARK_FSE_CONFIG: dict[str, Any] = {
"global_quant_config": {
"input_tensors": {
"dtype": "fp4",
"is_dynamic": True,
"qscheme": "per_group",
"ch_axis": -1,
"group_size": 32,
"block_size": None,
"symmetric": None,
"round_method": "half_even",
"scale_type": "float",
"scale_format": "e8m0",
"scale_calculation_mode": "even",
"mx_element_dtype": None,
"observer_cls": "PerBlockMXObserver",
"is_scale_quant": False,
"enable_buffer_reuse": False,
"max_input_numel": 4194304,
},
"output_tensors": None,
"weight": {
"dtype": "fp4",
"is_dynamic": False,
"qscheme": "per_group",
"ch_axis": -1,
"group_size": 32,
"block_size": None,
"symmetric": None,
"round_method": "half_even",
"scale_type": "float",
"scale_format": "e8m0",
"scale_calculation_mode": "even",
"mx_element_dtype": None,
"observer_cls": "PerBlockMXObserver",
"is_scale_quant": False,
"enable_buffer_reuse": False,
"max_input_numel": 4194304,
},
"bias": None,
"target_device": None,
},
"algo_config": None,
"softmax_quant_spec": None,
"quant_method": "quark",
"layer_type_quant_config": {},
"layer_quant_config": {},
"kv_cache_quant_config": {},
"kv_cache_post_rope": False,
"quant_mode": "eager_mode",
"version": "0.12+9d3d471cdf1",
"export": {
"kv_cache_group": [],
"min_kv_scale": 0.0,
"pack_method": "reorder",
"weight_format": "real_quantized",
"weight_merge_groups": None,
},
}
def get_deepseek_v4_quark_config(exclude: list[str]) -> dict[str, Any]:
"""Return the DeepSeek-V4-Pro-MXFP4 FSE quantization layout."""
# Mimics https://huggingface.co/amd/DeepSeek-V4-Pro-MXFP4.
quantization_config: dict[str, Any] = deepcopy(_QUARK_FSE_CONFIG)
quantization_config["exclude"] = exclude
mxfp4_config = cast(
dict[str, Any], deepcopy(quantization_config["global_quant_config"])
)
fp8_config = deepcopy(mxfp4_config)
fp8_config["input_tensors"].update(
{
"dtype": "fp8_e4m3",
"group_size": 128,
"symmetric": True,
"scale_type": None,
"scale_format": None,
"scale_calculation_mode": None,
"observer_cls": "PerGroupMinMaxObserver",
}
)
fp8_config["weight"].update(
{
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"ch_axis": None,
"group_size": None,
"block_size": [128, 128],
"symmetric": True,
"scale_type": "float8_e8m0fnu",
"scale_format": None,
"scale_calculation_mode": None,
"observer_cls": "PerBlock2DMinMaxObserver",
}
)
quantization_config["layer_quant_config"] = {
r"re:mtp\.0\.ffn\.experts\.\d+\.w1": mxfp4_config,
r"re:mtp\.0\.ffn\.shared_experts\.w1": fp8_config,
}
return quantization_config
def get_fse_test_model_config(
model_type: str,
quantization_config: dict[str, Any],
) -> tuple[object, type[nn.Module]]:
if model_type == "minimax_m3":
config = MiniMaxM3TextConfig(
hidden_size=128,
intermediate_size=32,
dense_intermediate_size=32,
num_hidden_layers=2,
num_attention_heads=1,
num_key_value_heads=1,
head_dim=128,
num_local_experts=2,
num_experts_per_tok=1,
moe_layer_freq=[0, 1],
sparse_attention_config=None,
rotary_dim=64,
quantization_config=quantization_config,
)
return config, minimax_m3_model.MiniMaxM3Model
if model_type == "deepseek_v4":
from vllm.models.deepseek_v4.amd.model import DeepseekV4Model
config = SimpleNamespace(
vocab_size=256,
hidden_size=128,
num_hidden_layers=1,
num_attention_heads=1,
head_dim=128,
max_position_embeddings=128,
sliding_window=None,
compress_ratios=[1],
rope_theta=10000.0,
compress_rope_theta=10000.0,
rope_parameters={"rope_type": "default"},
rms_norm_eps=1e-6,
hidden_act="silu",
q_lora_rank=0,
o_lora_rank=0,
o_groups=1,
qk_rope_head_dim=64,
index_head_dim=64,
index_n_heads=1,
index_topk=1,
n_routed_experts=2,
n_shared_experts=1,
num_experts_per_tok=1,
n_group=1,
topk_group=1,
moe_intermediate_size=32,
norm_topk_prob=False,
num_hash_layers=0,
swiglu_limit=0.0,
hc_eps=1e-6,
hc_mult=1,
hc_sinkhorn_iters=1,
expert_dtype="fp4",
num_nextn_predict_layers=1,
quantization_config=quantization_config,
)
return config, DeepseekV4Model
if model_type == "qwen3_5":
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
config = Qwen3_5MoeTextConfig(
vocab_size=256,
hidden_size=128,
num_hidden_layers=1,
num_attention_heads=1,
num_key_value_heads=1,
head_dim=128,
linear_key_head_dim=64,
linear_value_head_dim=64,
linear_num_key_heads=1,
linear_num_value_heads=1,
moe_intermediate_size=32,
shared_expert_intermediate_size=32,
num_experts_per_tok=1,
num_experts=2,
layer_types=["full_attention"],
quantization_config=quantization_config,
)
return config, Qwen3_5Model
if model_type == "deepseek_v2":
from transformers import DeepseekV2Config
from vllm.model_executor.models.deepseek_v2 import DeepseekV2Model
config = DeepseekV2Config(
vocab_size=256,
hidden_size=128,
intermediate_size=32,
num_hidden_layers=1,
num_attention_heads=1,
num_key_value_heads=1,
head_dim=128,
first_k_dense_replace=0,
n_routed_experts=2,
n_shared_experts=1,
num_experts_per_tok=1,
n_group=1,
topk_group=1,
moe_intermediate_size=32,
q_lora_rank=None,
kv_lora_rank=0,
qk_nope_head_dim=0,
qk_rope_head_dim=0,
v_head_dim=0,
quantization_config=quantization_config,
)
return config, DeepseekV2Model
if model_type == "glm4_moe":
from transformers.models.glm4_moe import Glm4MoeConfig
from vllm.model_executor.models.glm4_moe import Glm4MoeModel
config = Glm4MoeConfig(
vocab_size=256,
hidden_size=128,
intermediate_size=32,
num_hidden_layers=1,
num_attention_heads=1,
num_key_value_heads=1,
head_dim=128,
moe_intermediate_size=32,
n_routed_experts=2,
n_shared_experts=1,
num_experts_per_tok=1,
first_k_dense_replace=0,
quantization_config=quantization_config,
)
return config, Glm4MoeModel
raise ValueError(f"Unsupported FSE test model: {model_type}")
def test_determine_expert_counts_fuse_shared_experts_override(
monkeypatch: pytest.MonkeyPatch,
) -> None:
common_args = (8, 0, 2)
assert determine_expert_counts(*common_args, True)[2] == 2
assert determine_expert_counts(*common_args, False)[2] == 0
def test_resolve_layer_fused_shared_expert_skips_compatibility_when_disabled(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
fused_moe_utils.rocm_aiter_ops,
"is_fusion_moe_shared_experts_enabled",
lambda: False,
)
monkeypatch.setattr(
fused_moe_utils,
"is_shared_expert_quant_fse_compatible",
lambda *_: pytest.fail(
"compatibility must not be checked when FSE is disabled"
),
)
assert not fused_moe_utils.resolve_layer_fused_shared_expert(
object(), "model.layers.0.mlp"
)
def test_resolve_layer_fused_shared_expert_normalizes_unavailable_aiter(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
fused_moe_utils.rocm_aiter_ops,
"is_fusion_moe_shared_experts_enabled",
lambda: None,
)
assert (
fused_moe_utils.resolve_layer_fused_shared_expert(
object(), "model.layers.0.mlp"
)
is False
)
def test_resolve_layer_fused_shared_expert_passes_module_prefixes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
quant_config = object()
monkeypatch.setattr(
fused_moe_utils.rocm_aiter_ops,
"is_fusion_moe_shared_experts_enabled",
lambda: True,
)
calls: list[tuple[object, str, str]] = []
def check_compatibility(
config: object, expert_prefix: str, shared_expert_prefix: str
) -> tuple[bool, str | None]:
calls.append((config, expert_prefix, shared_expert_prefix))
return True, None
monkeypatch.setattr(
fused_moe_utils, "is_shared_expert_quant_fse_compatible", check_compatibility
)
assert fused_moe_utils.resolve_layer_fused_shared_expert(
quant_config,
"model.layers.0.mlp",
shared_expert_name="shared_expert",
)
assert calls == [
(
quant_config,
"model.layers.0.mlp.experts",
"model.layers.0.mlp.shared_expert",
)
]
def test_resolve_layer_fused_shared_expert_rejects_incompatible_quantization(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
monkeypatch.setattr(
fused_moe_utils.rocm_aiter_ops,
"is_fusion_moe_shared_experts_enabled",
lambda: True,
)
monkeypatch.setattr(
fused_moe_utils,
"is_shared_expert_quant_fse_compatible",
lambda *_: (False, "shared experts are excluded"),
)
assert not fused_moe_utils.resolve_layer_fused_shared_expert(
object(), "model.layers.0.mlp"
)
assert "shared experts are excluded" in caplog.text
def test_deepseek_v4_shared_expert_fse_uses_mtp_quantization_config_prefix(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class DeepseekV4Config:
expert_dtype = "fp4"
def _is_quark_mxfp4_ocp(self, hf_config: object) -> bool:
return True
hf_config = SimpleNamespace(
num_hidden_layers=2,
quantization_config={
"layer_quant_config": {
r"re:mtp\.0\.ffn\.shared_experts\.w1": {"weight": {"dtype": "fp4"}}
},
"global_quant_config": {"weight": {"dtype": "fp8"}},
},
)
monkeypatch.setattr(
deepseek_v4_quant_config, "DeepseekV4FP8Config", DeepseekV4Config
)
monkeypatch.setattr(
vllm_config_module,
"get_current_vllm_config",
lambda: SimpleNamespace(model_config=SimpleNamespace(hf_config=hf_config)),
)
compatible, reason = is_shared_expert_quant_fse_compatible(
DeepseekV4Config(),
"model.layers.2.ffn.experts",
"model.layers.2.ffn.shared_experts",
)
assert compatible
assert reason is None
def test_is_model_fused_shared_expert_compatible() -> None:
class MoE(nn.Module):
def __init__(self, enabled: bool) -> None:
super().__init__()
self.is_fused_shared_expert_enabled = enabled
class Layer(nn.Module):
def __init__(self, enabled: bool) -> None:
super().__init__()
self.mlp = MoE(enabled)
enabled_layers = nn.ModuleList([Layer(True)])
disabled_layers = nn.ModuleList([Layer(False)])
mixed_layers = nn.ModuleList([Layer(True), Layer(False)])
empty_layers = nn.ModuleList()
pipeline_layers = nn.ModuleList([Layer(True), PPMissingLayer()])
assert fused_moe_utils.is_model_fused_shared_expert_compatible(
enabled_layers, MoE, "mlp"
)
assert not fused_moe_utils.is_model_fused_shared_expert_compatible(
disabled_layers, MoE, "mlp"
)
assert not fused_moe_utils.is_model_fused_shared_expert_compatible(
empty_layers, MoE, "mlp"
)
assert fused_moe_utils.is_model_fused_shared_expert_compatible(
pipeline_layers, MoE, "mlp"
)
with pytest.raises(NotImplementedError, match="1 enabled and 1 disabled layers"):
fused_moe_utils.is_model_fused_shared_expert_compatible(
mixed_layers, MoE, "mlp"
)
@pytest.mark.parametrize(
"model_type",
["minimax_m3", "deepseek_v4", "qwen3_5", "glm4_moe", "deepseek_v2"],
)
@pytest.mark.parametrize(
("use_fse", "exclude"),
[
(False, []),
(True, []),
(True, ["*.shared_experts.*"]),
],
)
def test_models_fse_init(
model_type: str,
use_fse: bool,
exclude: list[str],
dist_init: None,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Model construction resolves FSE consistently with Quark quantization."""
quantization_config: dict[str, Any] = (
get_deepseek_v4_quark_config(["layers.0.ffn.shared_experts"] if exclude else [])
if model_type == "deepseek_v4"
else {**_QUARK_FSE_CONFIG, "exclude": exclude}
)
config, model_constructor = get_fse_test_model_config(
model_type, quantization_config
)
vllm_config = VllmConfig()
vllm_config.model_config = SimpleNamespace(
hf_config=config,
hf_text_config=config,
dtype=torch.bfloat16,
max_model_len=128,
is_diffusion=False,
head_dtype=None,
is_mm_prefix_lm=False,
multimodal_config=None,
quantization_config=None,
)
vllm_config.parallel_config.enable_expert_parallel = False
if model_type == "deepseek_v4":
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
vllm_config.cache_config.cache_dtype = "fp8_ds_mla"
vllm_config.quant_config = DeepseekV4FP8Config(
is_checkpoint_fp8_serialized=True,
weight_block_size=[128, 128],
)
else:
vllm_config.quant_config = QuarkConfig(quantization_config)
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
if model_type == "deepseek_v4":
from vllm.models.deepseek_v4.amd import model as deepseek_v4_model
warning_logger = deepseek_v4_model.logger
elif model_type == "minimax_m3":
warning_logger = minimax_m3_model.logger
else:
warning_logger = fused_moe_utils.logger
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_USE_AITER", str(use_fse))
mp.setenv("VLLM_ROCM_USE_AITER_MOE", str(use_fse))
mp.setenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", str(use_fse))
mp.setattr(
"vllm.model_executor.layers.fused_moe.experts."
"ocp_mx_emulation_moe.has_quark",
lambda: True,
)
importlib.reload(envs)
mp.setattr(envs, "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", use_fse)
rocm_aiter_ops.refresh_env_variables()
aiter_fse_enabled = bool(rocm_aiter_ops.is_fusion_moe_shared_experts_enabled())
# These AMD-specific models currently use the raw environment flag.
# and do not rely on `rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()`.
fse_enabled = use_fse and (
aiter_fse_enabled or model_type in {"deepseek_v4", "minimax_m3"}
)
with (
patch.object(warning_logger, "warning") as warning,
set_current_vllm_config(vllm_config),
):
model = model_constructor(vllm_config=vllm_config)
mtp = None
if model_type == "deepseek_v4" and use_fse and not exclude:
from vllm.models.deepseek_v4.amd.mtp import (
DeepSeekV4MTP,
)
vllm_config.speculative_config = SimpleNamespace(
draft_model_config=SimpleNamespace(hf_config=config)
)
mtp = DeepSeekV4MTP(vllm_config=vllm_config)
assert model.is_fused_shared_expert_enabled is (fse_enabled and not exclude)
# The dummy quant config here uses mixed mxfp4/fp8 for experts/shared_expert
# so should just raise a warning.
if mtp is not None:
assert not mtp.model.layers[
"1"
].mtp_block.ffn.is_fused_shared_expert_enabled
warning.assert_called_once()
assert (
"DeepSeek-V4 shared experts at mtp.0.ffn.shared_experts"
in (warning.call_args.args[1])
)
if aiter_fse_enabled and exclude:
warning.assert_called_once()
assert (
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled"
in (warning.call_args.args[0])
)
assert "excludes shared experts" in warning.call_args.args[1]
importlib.reload(envs)
rocm_aiter_ops.refresh_env_variables()
@pytest.mark.parametrize(
("exclude", "expected"),
[([], True), (["*.shared_expert.*"], False)],
)
def test_quark_shared_expert_fse_compatibility(
exclude: list[str], expected: bool
) -> None:
compatible, reason = is_shared_expert_quant_fse_compatible(
QuarkConfig(
{
"exclude": exclude,
"global_quant_config": {},
"layer_quant_config": {},
}
),
"model.layers.0.mlp.experts",
"model.layers.0.mlp.shared_expert",
)
assert compatible is expected
if expected:
assert reason is None
else:
assert (
reason
== "Quark excludes shared experts at model.layers.0.mlp.shared_expert"
)
def test_quark_shared_expert_fse_requires_matching_layer_quant_configs() -> None:
global_quant_config = {"weight": {"dtype": "fp4"}}
quant_config = QuarkConfig(
{
"exclude": [],
"global_quant_config": global_quant_config,
"layer_quant_config": {
"model.layers.0.mlp.shared_expert.gate_up_proj": {
"weight": {"dtype": "fp8"}
},
"model.layers.0.mlp.shared_expert.down_proj": {
"weight": {"dtype": "fp8"}
},
},
}
)
compatible, reason = is_shared_expert_quant_fse_compatible(
quant_config,
"model.layers.0.mlp.experts",
"model.layers.0.mlp.shared_expert",
)
assert not compatible
assert reason == (
"Quark uses different quantization configurations for routed and "
"shared experts at model.layers.0.mlp.shared_expert"
)
def test_quark_shared_expert_fse_rejects_partial_packed_projection_override() -> None:
fp4_config = {"weight": {"dtype": "fp4"}}
fp8_config = {"weight": {"dtype": "fp8"}}
quant_config = QuarkConfig(
{
"exclude": [],
"global_quant_config": fp4_config,
"layer_quant_config": {
"model.layers.0.mlp.experts": fp8_config,
"model.layers.0.mlp.shared_expert.gate_proj": fp8_config,
"model.layers.0.mlp.shared_expert.down_proj": fp8_config,
},
}
)
quant_config.packed_modules_mapping = {"gate_up_proj": ["gate_proj", "up_proj"]}
compatible, reason = is_shared_expert_quant_fse_compatible(
quant_config,
"model.layers.0.mlp.experts",
"model.layers.0.mlp.shared_expert",
)
assert not compatible
assert reason == (
"Quark uses different quantization configurations for routed and "
"shared experts at model.layers.0.mlp.shared_expert"
)
def test_quark_layer_config_from_name_checks_packed_projections() -> None:
quant_config = QuarkConfig(
{
"global_quant_config": {"weight": {"dtype": "fp4"}},
"layer_quant_config": {
"model.layers.0.mlp.shared_expert.w1": {"weight": {"dtype": "fp4"}},
"model.layers.0.mlp.shared_expert.w3": {"weight": {"dtype": "fp8"}},
},
}
)
quant_config.packed_modules_mapping = {"gate_up_proj": ["w1", "w3"]}
with pytest.raises(ValueError, match="requires all to use the same scheme"):
quant_config.get_layer_quant_config_from_name(
"model.layers.0.mlp.shared_expert.gate_up_proj"
)
def test_quark_packed_layer_config_must_match_global_config() -> None:
quant_config = QuarkConfig(
{
"global_quant_config": {"weight": {"dtype": "fp4"}},
"layer_type_quant_config": {},
"layer_quant_config": {
"model.layers.0.mlp.shared_expert.w1": {"weight": {"dtype": "fp8"}},
},
}
)
quant_config.packed_modules_mapping = {"gate_up_proj": ["w1", "w3"]}
with pytest.raises(ValueError, match="requires all to use the same scheme"):
quant_config._find_matched_config(
"model.layers.0.mlp.shared_expert.gate_up_proj", nn.Module()
)
def test_non_quark_shared_expert_fse_is_incompatible() -> None:
compatible, reason = is_shared_expert_quant_fse_compatible(
object(),
"model.layers.0.mlp.experts",
"model.layers.0.mlp.shared_experts",
)
assert not compatible
assert reason == (
"shared-expert FSE quantization compatibility is not implemented for object"
)
+5 -14
View File
@@ -6,7 +6,6 @@ from typing import Any
import torch
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import ParallelConfig, get_current_vllm_config
from vllm.distributed import (
@@ -74,23 +73,13 @@ def determine_expert_counts(
num_experts: int,
num_redundant_experts: int,
n_shared_experts: int | None,
is_act_and_mul: bool,
fuse_shared_experts: bool,
) -> tuple[int, int, int]:
global_num_experts = num_experts + num_redundant_experts
logical_num_experts = num_experts
# Shared-expert fusion: append the shared expert(s) as routed-expert slots
# so they run in the same grouped GEMM. Gated by
# VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: either the native aiter fused-MoE
# path (env + master switch, via is_fusion_moe_shared_experts_enabled) or the
# backend-neutral router-append path (env alone, independent of the master
# switch; e.g. the MM3 triton/flydsl mxfp8 MoE). Gated activations only.
fuse_shared_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
or envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
) and is_act_and_mul
num_fused_shared_experts = (
n_shared_experts if n_shared_experts is not None and fuse_shared_enabled else 0
n_shared_experts if n_shared_experts is not None and fuse_shared_experts else 0
)
return global_num_experts, logical_num_experts, num_fused_shared_experts
@@ -132,6 +121,7 @@ def FusedMoEFactory(
ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"),
is_fused_checkpoint_transposed: bool = False,
n_shared_experts: int | None = None,
fuse_shared_experts: bool = False,
router_logits_dtype: torch.dtype | None = None,
gate: torch.nn.Module | None = None,
shared_experts: torch.nn.Module | None = None,
@@ -198,6 +188,7 @@ def FusedMoEFactory(
block scales use transposed storage.
n_shared_experts: Number of shared experts to fuse into the routed
grouped GEMM (ROCm; requires aiter FSE or the router-append path)
fuse_shared_experts: Whether to enable shared-expert fusion.
router_logits_dtype: Data type for router logits buffers
gate: Pre-configured gate module
shared_experts: Pre-configured shared experts module
@@ -244,7 +235,7 @@ def FusedMoEFactory(
num_experts,
num_redundant_experts,
n_shared_experts,
is_act_and_mul,
fuse_shared_experts,
)
)
@@ -1,15 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from collections.abc import Iterable
from math import prod
from typing import TYPE_CHECKING
import torch
import torch.nn.functional as F
from torch import nn
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.config_utils import (
is_shared_expert_quant_fse_compatible,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
@@ -34,16 +40,95 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
per_tensor_dequantize,
)
from vllm.model_executor.models.utils import PPMissingLayer
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
logger = init_logger(__name__)
def resolve_layer_fused_shared_expert(
quant_config: "QuantizationConfig | None",
prefix: str,
shared_expert_name: str = "shared_experts",
) -> bool:
"""Resolve whether AITER fused shared-expert execution is enabled.
Args:
quant_config: Model quantization configuration.
prefix: MoE module prefix.
shared_expert_name: Shared-expert module name under ``prefix``.
Returns:
Whether AITER fused shared experts are enabled.
Raises:
ValueError: If requested shared-expert fusion is quantization-incompatible.
"""
# NOTE: is_fusion_moe_shared_experts_enabled is decorated with @if_aiter_supported
# that returns None if AITER is not available.
fse_requested = bool(rocm_aiter_ops.is_fusion_moe_shared_experts_enabled())
fse_compatible, fse_reason = (
is_shared_expert_quant_fse_compatible(
quant_config,
f"{prefix}.experts",
f"{prefix}.{shared_expert_name}",
)
if fse_requested
else (True, None)
)
is_fused_shared_expert_enabled = fse_requested and fse_compatible
if fse_requested and not is_fused_shared_expert_enabled:
logger.warning(
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
"cannot be enabled: %s.",
fse_reason,
)
return is_fused_shared_expert_enabled
def is_model_fused_shared_expert_compatible(
layers: nn.ModuleList | Iterable[nn.Module],
moe_cls: type[nn.Module],
moe_name: str,
) -> bool:
"""Resolve one fused-shared-expert state for a model's MoE layers."""
def get_moe_layer(layer: nn.Module) -> nn.Module | None:
for name in moe_name.split("."):
layer = getattr(layer, name, None)
if layer is None:
return None
return layer
moe_layers = (
moe_layer
for layer in layers
if not isinstance(layer, PPMissingLayer)
and (moe_layer := get_moe_layer(layer)) is not None
and isinstance(moe_layer, moe_cls)
)
enabled = [
getattr(layer, "is_fused_shared_expert_enabled", False) for layer in moe_layers
]
enabled_count = sum(enabled)
disabled_count = len(enabled) - enabled_count
if enabled_count > 0 and disabled_count > 0:
raise NotImplementedError(
"Fused shared experts must be enabled for all MoE layers; found "
f"{enabled_count} enabled and {disabled_count} disabled layers. "
"Per-layer fused shared experts is not yet supported. Please open "
"an issue."
)
return enabled_count > 0 and disabled_count == 0
@triton.jit
def _count_expert_num_tokens(
topk_ids_ptr,
@@ -576,63 +576,91 @@ class QuarkConfig(QuantizationConfig):
return True
def _find_matched_config(
self, layer_name: str, module: torch.nn.Module
) -> dict[str, Any]:
def get_layer_quant_config_from_name(
self, layer_name: str
) -> dict[str, Any] | None:
proj_name = layer_name.split(".")[-1]
if proj_name in self.packed_modules_mapping:
shard_proj_names = self.packed_modules_mapping[proj_name]
# Convert fused_name --> [shard_names]
shard_names = [
layer_name.replace(proj_name, shard_proj_name)
for shard_proj_name in shard_proj_names
]
shard_configs = []
for shard_name in shard_names:
if shard_name == layer_name:
config = cast(
dict[str, Any], self.quant_config.get("global_quant_config")
)
for shard_proj_name in shard_proj_names:
shard_name = layer_name.replace(proj_name, shard_proj_name)
if shard_name != layer_name:
config = self.get_layer_quant_config_from_name(shard_name)
else:
config = self._find_matched_config(shard_name, module)
config = None
shard_configs.append(config)
if not all(
deep_compare(q_config, shard_configs[0]) for q_config in shard_configs
matched_configs = [config for config in shard_configs if config is not None]
if matched_configs and not all(
deep_compare(config, matched_configs[0]) for config in matched_configs
):
raise ValueError(
f"Found a different quantization configuration for "
f"{shard_proj_names} in {layer_name}. vLLM "
"requires all to use the same scheme."
)
return shard_configs[0]
if matched_configs:
return matched_configs[0]
return None
else:
layer_quant_config = cast(
dict[str, Any], self.quant_config.get("layer_quant_config")
dict[str, Any], self.quant_config.get("layer_quant_config") or {}
)
def _matches_pattern(layer_name, pattern):
if "*" not in pattern:
return layer_name in pattern
return fnmatch.fnmatch(layer_name, pattern)
for name_pattern, config in layer_quant_config.items():
if _matches_pattern(layer_name, name_pattern):
if "*" not in name_pattern:
matches = layer_name in name_pattern
else:
matches = fnmatch.fnmatch(layer_name, name_pattern)
if matches:
return config
return None
layer_type = cast(str, type(module))
layer_type_quant_config = cast(
dict[str, Any], self.quant_config.get("layer_type_quant_config")
)
if layer_type in layer_type_quant_config:
return layer_type_quant_config[layer_type]
def _find_matched_config(
self, layer_name: str, module: torch.nn.Module
) -> dict[str, Any]:
# Priority order:
# 1. layer_quant_config,
# 2. layer_type_quant_config,
# 3. global_quant_config.
global_quant_config = cast(
dict[str, Any], self.quant_config.get("global_quant_config")
)
return global_quant_config
layer_type = cast(str, type(module))
layer_type_quant_config = cast(
dict[str, Any], self.quant_config.get("layer_type_quant_config")
)
global_quant_config = cast(
dict[str, Any], self.quant_config.get("global_quant_config")
)
fallback_config = layer_type_quant_config.get(layer_type, global_quant_config)
proj_name = layer_name.split(".")[-1]
if proj_name in self.packed_modules_mapping:
shard_proj_names = self.packed_modules_mapping[proj_name]
shard_configs = []
for shard_proj_name in shard_proj_names:
shard_name = layer_name.replace(proj_name, shard_proj_name)
if shard_name == layer_name:
config = fallback_config
else:
config = self.get_layer_quant_config_from_name(shard_name)
if config is None:
config = fallback_config
shard_configs.append(config)
if not all(
deep_compare(config, shard_configs[0]) for config in shard_configs
):
raise ValueError(
f"Found a different quantization configuration for "
f"{shard_proj_names} in {layer_name}. vLLM requires all "
"to use the same scheme."
)
return shard_configs[0]
else:
layer_quant_config = self.get_layer_quant_config_from_name(layer_name)
if layer_quant_config is not None:
return layer_quant_config
return fallback_config
def _get_scheme_from_config(
self, config: dict[str, Any], dynamic_mxfp4_quant: bool = False
@@ -0,0 +1,155 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
import regex as re
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
def is_shared_expert_quant_fse_compatible(
quant_config: "QuantizationConfig | None",
expert_prefix: str,
shared_expert_prefix: str,
projection_names: list[str] | None = None,
) -> tuple[bool, str | None]:
"""Check whether quantization permits fused shared-expert execution.
Args:
quant_config: Model quantization configuration.
expert_prefix: Routed-expert module prefix.
shared_expert_prefix: Shared-expert module prefix.
projection_names: Shared-expert projection names.
Returns:
A compatibility flag and, when incompatible, the reason.
"""
if projection_names is None:
projection_names = ["gate_up_proj", "down_proj"]
if quant_config is None:
return True, None
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
if isinstance(quant_config, DeepseekV4FP8Config):
from vllm.config import get_current_vllm_config
from vllm.model_executor.models.utils import extract_layer_index
if quant_config.expert_dtype != "fp4":
return False, "DeepSeek-V4 routed experts are not MXFP4"
hf_config = get_current_vllm_config().model_config.hf_config
# TODO: This is adapted from former `_shared_experts_are_fp4`, and
# needs to be cleaned up this . There should not be Quark-specific
# logic in DeepseekV4FP8Config.
quantization_config = getattr(hf_config, "quantization_config", None)
if quantization_config is None:
return False, "DeepSeek-V4 has no quantization configuration"
if not quant_config._is_quark_mxfp4_ocp(quantization_config):
return False, "DeepSeek-v4 FSE is only implemented/tested with Quark MXFP4"
layer_idx = extract_layer_index(shared_expert_prefix)
if layer_idx >= hf_config.num_hidden_layers:
shared_expert_prefix = (
f"mtp.{layer_idx - hf_config.num_hidden_layers}.ffn.shared_experts"
)
else:
shared_expert_prefix = f"layers.{layer_idx}.ffn.shared_experts"
if any(
entry.startswith(shared_expert_prefix)
for entry in quantization_config.get("exclude") or []
if isinstance(entry, str)
):
return (
False,
f"DeepSeek-V4 excludes shared experts at {shared_expert_prefix}",
)
shared_expert_weight_name = f"{shared_expert_prefix}.w1"
layer_quant_config = quantization_config.get("layer_quant_config") or {}
layer_config = layer_quant_config.get(shared_expert_weight_name)
if layer_config is None:
layer_config = next(
(
config
for pattern, config in layer_quant_config.items()
if isinstance(pattern, str)
and pattern.startswith("re:")
and re.fullmatch(
pattern.removeprefix("re:"), shared_expert_weight_name
)
),
None,
)
shared_weight_config = (
layer_config or quantization_config.get("global_quant_config") or {}
).get("weight") or {}
if shared_weight_config.get("dtype") == "fp4":
return True, None
return (
False,
f"DeepSeek-V4 shared experts at {shared_expert_prefix} are not MXFP4",
)
if isinstance(quant_config, QuarkConfig):
# TODO: layer_type_quant_config is not taken into account here.
assert "exclude" in quant_config.quant_config
assert "global_quant_config" in quant_config.quant_config
is_compatible = not any(
"shared_expert" in str(entry)
for entry in quant_config.quant_config["exclude"]
)
if not is_compatible:
return False, f"Quark excludes shared experts at {shared_expert_prefix}"
global_quant_config = quant_config.quant_config["global_quant_config"]
def get_projection_quant_configs(layer_name: str) -> list[object]:
module_prefix, _, projection_name = layer_name.rpartition(".")
packed_projection_names = quant_config.packed_modules_mapping.get(
projection_name, [projection_name]
)
return [
quant_config.get_layer_quant_config_from_name(
f"{module_prefix}.{packed_projection_name}"
)
or global_quant_config
for packed_projection_name in packed_projection_names
]
expert_quant_config = (
quant_config.get_layer_quant_config_from_name(expert_prefix)
or global_quant_config
)
shared_expert_quant_configs = [
config
for projection_name in projection_names
for config in get_projection_quant_configs(
f"{shared_expert_prefix}.{projection_name}"
)
]
if all(config == expert_quant_config for config in shared_expert_quant_configs):
return True, None
return (
False,
"Quark uses different quantization configurations for routed and "
f"shared experts at {shared_expert_prefix}",
)
# TODO: Extend FSE support detection to other quantization methods. Typically,
# one would check that the experts and shared_experts use the same
# quantization config. This may be refactored as part of QuantizationConfig later.
return (
False,
"shared-expert FSE quantization compatibility is not implemented for "
f"{type(quant_config).__name__}",
)
+22 -10
View File
@@ -46,6 +46,10 @@ from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -148,10 +152,14 @@ class AXK1MoE(nn.Module):
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
self.is_fusion_moe_shared_experts_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled:
self.is_fused_shared_expert_enabled = False
if config.n_shared_experts is not None:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config, prefix
)
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
@@ -189,8 +197,9 @@ class AXK1MoE(nn.Module):
num_redundant_experts=self.n_redundant_experts,
is_sequence_parallel=self.is_sequence_parallel,
n_shared_experts=config.n_shared_experts
if self.is_fusion_moe_shared_experts_enabled
if self.is_fused_shared_expert_enabled
else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -725,6 +734,12 @@ class AXK1Model(nn.Module):
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
AXK1MoE,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
@@ -789,9 +804,6 @@ class AXK1Model(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping: list[tuple[str, str, int | str]] = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "gate_proj", 0),
@@ -822,7 +834,7 @@ class AXK1Model(nn.Module):
num_experts=self.config.n_routed_experts
+ (
(self.config.n_shared_experts or 0)
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
num_redundant_experts=self.num_redundant_experts,
@@ -839,7 +851,7 @@ class AXK1Model(nn.Module):
continue # skip spec decode layers for main model
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
for param_name, weight_name, shard_id in stacked_params_mapping:
+10 -6
View File
@@ -7,13 +7,15 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import tensor_model_parallel_all_gather
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -255,6 +257,11 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
self.moe_mlp_layers.append(layer.mlp)
self.moe_layers.append(layer.mlp.experts)
self.extract_moe_parameters(example_moe)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.model.layers.values(),
DeepseekV2MoE,
"mtp_block.mlp",
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
@@ -285,9 +292,6 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
return self.model.compute_logits(hidden_states, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
@@ -310,7 +314,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
)
@@ -326,7 +330,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
if spec_layer is None:
continue
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
name = self._rewrite_spec_layer_name(spec_layer, name)
+22 -10
View File
@@ -54,6 +54,10 @@ from vllm.model_executor.layers.fused_moe import (
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -329,9 +333,13 @@ class DeepseekV2MoE(nn.Module):
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
self.is_fusion_moe_shared_experts_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
self.is_fused_shared_expert_enabled = False
if config.n_shared_experts is not None:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config, prefix
)
if (
self.is_rocm_aiter_moe_enabled
and self.gate.e_score_correction_bias is not None
@@ -340,7 +348,7 @@ class DeepseekV2MoE(nn.Module):
# Accumulates in fp32; avoids bf16->fp32 cast.
self.gate.set_out_dtype(self.gate.weight.dtype)
if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled:
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
@@ -377,8 +385,9 @@ class DeepseekV2MoE(nn.Module):
is_sequence_parallel=self.is_sequence_parallel,
reduce_results=reduce_results,
n_shared_experts=config.n_shared_experts
if self.is_fusion_moe_shared_experts_enabled
if self.is_fused_shared_expert_enabled
else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
router_logits_dtype=self.gate.out_dtype,
)
@@ -1401,6 +1410,12 @@ class DeepseekV2Model(nn.Module):
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
DeepseekV2MoE,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(self.hidden_size, eps=config.rms_norm_eps)
else:
@@ -1511,9 +1526,6 @@ class DeepseekV2Model(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping: list[tuple[str, str, int | str]] = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "gate_proj", 0),
@@ -1554,7 +1566,7 @@ class DeepseekV2Model(nn.Module):
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
num_redundant_experts=self.num_redundant_experts,
@@ -1582,7 +1594,7 @@ class DeepseekV2Model(nn.Module):
continue # this layer has no indexer; drop its checkpoint weights
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
if _try_load_fp8_indexer_wk(
+20 -12
View File
@@ -39,12 +39,15 @@ from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -72,8 +75,6 @@ from .utils import (
skip_spec_layers,
)
logger = init_logger(__name__)
class Glm4MoeMLP(nn.Module):
def __init__(
@@ -159,14 +160,15 @@ class Glm4MoE(nn.Module):
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
# AITER fused shared-expert (FSE) gate; mirrors the deepseek_v2.py
# pattern (see Glm4MoE / MoERunner wiring there).
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
self.is_fusion_moe_shared_experts_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled:
self.is_fused_shared_expert_enabled = False
if config.n_shared_experts is not None:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config, prefix
)
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
@@ -200,10 +202,9 @@ class Glm4MoE(nn.Module):
num_redundant_experts=self.n_redundant_experts,
router_logits_dtype=torch.float32,
n_shared_experts=(
config.n_shared_experts
if self.is_fusion_moe_shared_experts_enabled
else None
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
),
fuse_shared_experts=self.is_fused_shared_expert_enabled,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -443,6 +444,12 @@ class Glm4MoeModel(nn.Module):
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Glm4MoE,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
@@ -488,6 +495,7 @@ class Glm4MoeModel(nn.Module):
skip_spec_layers(weights, self.config),
n_routed_experts=self.config.n_routed_experts,
n_shared_experts=self.config.n_shared_experts or 1,
enabled=self.is_fused_shared_expert_enabled,
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+11 -6
View File
@@ -34,7 +34,6 @@ from torch import nn
if TYPE_CHECKING:
from transformers.models.glm4_moe_lite import Glm4MoeLiteConfig
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import (
@@ -44,6 +43,9 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -254,6 +256,12 @@ class Glm4MoeLiteModel(nn.Module):
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Glm4MoeLite,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
@@ -320,9 +328,6 @@ class Glm4MoeLiteModel(nn.Module):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "gate_proj", 0),
@@ -345,7 +350,7 @@ class Glm4MoeLiteModel(nn.Module):
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
)
@@ -361,7 +366,7 @@ class Glm4MoeLiteModel(nn.Module):
continue # skip spec decode layers for main model
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
for param_name, weight_name, shard_id in stacked_params_mapping:
@@ -30,12 +30,14 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.model_executor.layers.fused_moe import (
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -224,6 +226,11 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
self.moe_mlp_layers.append(layer.mlp)
self.moe_layers.append(layer.mlp.experts)
self.extract_moe_parameters(example_moe)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.model.layers.values(),
Glm4MoeLite,
"mtp_block.mlp",
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
@@ -250,9 +257,6 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
return self.model.compute_logits(hidden_states, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
@@ -268,7 +272,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
)
@@ -282,7 +286,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
if spec_layer is None:
continue
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
name = self._rewrite_spec_layer_name(spec_layer, name)
for param_name, weight_name, shard_id in stacked_params_mapping:
+10 -7
View File
@@ -31,12 +31,14 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import CacheConfig, ParallelConfig, VllmConfig
from vllm.model_executor.layers.fused_moe import (
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -212,6 +214,11 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
self.moe_mlp_layers.append(layer.mlp)
self.moe_layers.append(layer.mlp.experts)
self.extract_moe_parameters(example_moe)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.model.layers.values(),
Glm4MoE,
"mtp_block.mlp",
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
@@ -238,10 +245,6 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
return self.model.compute_logits(hidden_states, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# FSE weight loading mirrors glm4_moe.py / deepseek_mtp.py.
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
@@ -254,7 +257,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
num_experts = self.config.n_routed_experts
if rocm_aiter_moe_shared_expert_enabled and self.config.n_shared_experts:
if self.is_fused_shared_expert_enabled and self.config.n_shared_experts:
num_experts += self.config.n_shared_experts
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
@@ -279,7 +282,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
name = self._rewrite_spec_layer_name(spec_layer, name)
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
for param_name, weight_name, shard_id in stacked_params_mapping:
+9 -4
View File
@@ -29,13 +29,15 @@ from collections.abc import Iterable
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import (
get_pp_group,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as Qwen3_5RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
@@ -78,7 +80,6 @@ from .qwen3_next import (
Qwen3NextModel,
Qwen3NextSparseMoeBlock,
QwenNextMixtureOfExperts,
_is_shared_expert_fse_compatible,
)
from .qwen3_vl import (
Qwen3_VisionTransformer,
@@ -256,6 +257,11 @@ class Qwen3_5Model(Qwen3NextModel):
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers"
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen3NextSparseMoeBlock,
"mlp",
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
@@ -274,8 +280,7 @@ class Qwen3_5Model(Qwen3NextModel):
if "moe" in self.config.model_type:
weights = maybe_fuse_shared_experts(
weights,
enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
and _is_shared_expert_fse_compatible(self.quant_config),
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=self.config.num_experts,
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
+11 -7
View File
@@ -7,11 +7,13 @@ from collections.abc import Iterable
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -25,8 +27,8 @@ from vllm.model_executor.models.qwen3_5 import (
Qwen3_5RMSNorm,
)
from vllm.model_executor.models.qwen3_next import (
Qwen3NextSparseMoeBlock,
QwenNextMixtureOfExperts,
_is_shared_expert_fse_compatible,
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
@@ -122,6 +124,11 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
for idx in range(self.num_mtp_layers)
)
vllm_config.quant_config = original_quant
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen3NextSparseMoeBlock,
"mlp",
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
@@ -185,10 +192,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = maybe_fuse_shared_experts(
weights,
enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
and _is_shared_expert_fse_compatible(
get_current_vllm_config().quant_config
),
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=getattr(self.config, "num_experts", 0),
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
+22 -32
View File
@@ -8,7 +8,6 @@ from itertools import islice
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.distributed import (
@@ -18,9 +17,12 @@ from vllm.distributed import (
tensor_model_parallel_all_gather,
tensor_model_parallel_reduce_scatter,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate
from vllm.model_executor.layers.layernorm import (
GemmaRMSNorm as Qwen3NextRMSNorm,
@@ -73,8 +75,6 @@ from .utils import (
maybe_prefix,
)
logger = init_logger(__name__)
KVCache = tuple[torch.Tensor, torch.Tensor]
@@ -90,26 +90,6 @@ def _should_use_sequence_parallel(vllm_config: VllmConfig) -> bool:
)
def _is_shared_expert_fse_compatible(quant_config) -> bool:
"""Check if shared expert can be fused with routed experts.
FSE requires that shared and routed expert weights use the same
quantization format. Returns False when the shared expert is
excluded from quantization (e.g. float32 shared in an MXFP4 model)
or has a different quant spec than routed experts.
"""
if quant_config is None:
return True
# Quark stores its full config dict in quant_config.quant_config
raw_config = getattr(quant_config, "quant_config", None)
if not isinstance(raw_config, dict):
return True
exclude = raw_config.get("exclude", [])
if not exclude:
return True
return not any("shared_expert." in str(e) for e in exclude)
class Qwen3NextSparseMoeBlock(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -157,15 +137,18 @@ class Qwen3NextSparseMoeBlock(nn.Module):
prefix=f"{prefix}.shared_expert_gate",
)
_fse_requested = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
_fse_enabled = _fse_requested and _is_shared_expert_fse_compatible(quant_config)
if _fse_requested and not _fse_enabled:
logger.warning(
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
"shared expert has a different quantization spec than routed "
"experts. Falling back to non-fused shared expert path."
self.is_fused_shared_expert_enabled = False
if config.shared_expert_intermediate_size > 0:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config,
prefix,
shared_expert_name="shared_expert",
)
if _fse_enabled or config.shared_expert_intermediate_size <= 0:
if (
self.is_fused_shared_expert_enabled
or config.shared_expert_intermediate_size <= 0
):
self.shared_expert = None
else:
self.shared_expert = Qwen3NextMLP(
@@ -193,6 +176,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
num_redundant_experts=self.n_redundant_experts,
is_sequence_parallel=self.is_sequence_parallel,
n_shared_experts=1 if self.shared_expert is None else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
shared_expert_gate=self.shared_expert_gate
if self.shared_expert is None
else None,
@@ -599,6 +583,11 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers"
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen3NextSparseMoeBlock,
"mlp",
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
@@ -678,6 +667,7 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = maybe_fuse_shared_experts(
weights,
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=getattr(self.config, "num_experts", 0),
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
@@ -11,6 +11,9 @@ from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -21,6 +24,7 @@ from vllm.model_executor.models.qwen3_next import (
Qwen3NextDecoderLayer,
Qwen3NextModel,
Qwen3NextRMSNorm,
Qwen3NextSparseMoeBlock,
QwenNextMixtureOfExperts,
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
@@ -91,6 +95,11 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
for idx in range(self.num_mtp_layers)
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen3NextSparseMoeBlock,
"mlp",
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
@@ -155,6 +164,7 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = maybe_fuse_shared_experts(
weights,
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=self.config.num_experts,
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
+10 -6
View File
@@ -8,12 +8,14 @@ from collections.abc import Callable, Iterable
import torch
import torch.nn as nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.distributed import tensor_model_parallel_all_reduce
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -167,6 +169,11 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.set_moe_parameters()
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.model.layers.values(),
DeepseekV2MoE,
"mtp_block.mlp",
)
def set_moe_parameters(self):
self.num_moe_layers = self.config.num_nextn_predict_layers
@@ -231,9 +238,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
return name
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
@@ -250,7 +254,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
)
@@ -266,7 +270,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
if spec_layer is None:
continue
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
name = self._rewrite_spec_layer_name(spec_layer, name)
+3 -43
View File
@@ -7,7 +7,6 @@ import torch
import torch.nn as nn
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.fused_embed_norm import (
@@ -344,9 +343,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
return name
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
@@ -360,12 +356,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
else 0
),
num_experts=self.config.n_routed_experts,
)
pp_missing_layer_names = get_pp_missing_layer_names(self)
@@ -378,9 +369,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
if spec_layer is None:
continue
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
)
name = self._rewrite_spec_layer_name(spec_layer, name)
if _try_load_fp8_indexer_wk(
@@ -398,8 +386,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
continue
if ("mlp.experts." in name) and name not in params_dict:
continue
if is_fusion_moe_shared_experts_layer:
continue
name_mapped = name.replace(weight_name, param_name)
if (
param_name == "fused_qkv_a_proj"
@@ -415,32 +401,10 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
break
else:
num_chunks = 1
if is_fusion_moe_shared_experts_layer:
num_chunks = getattr(self.config, "n_shared_experts", 1) or 1
split_dim = (
1
if ("down_proj.weight" in name and loaded_weight.ndim > 1)
else 0
)
total = loaded_weight.shape[split_dim]
assert total % num_chunks == 0
chunk_size = total // num_chunks
for j in range(num_chunks):
chunk_name = name
weight_to_load = loaded_weight
if is_fusion_moe_shared_experts_layer:
chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size)
if loaded_weight.ndim == 1:
weight_to_load = loaded_weight[chunk_slice]
elif split_dim == 0:
weight_to_load = loaded_weight[chunk_slice, :]
else:
weight_to_load = loaded_weight[:, chunk_slice]
chunk_name = name.replace(
"mlp.shared_experts",
f"mlp.experts.{self.config.n_routed_experts + j}",
)
is_expert_weight = False
for mapping in expert_params_mapping:
@@ -462,10 +426,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
return_success=True,
)
if success:
if not is_fusion_moe_shared_experts_layer:
name = name_mapped
else:
loaded_params.add(name_mapped)
name = name_mapped
break
else:
if is_expert_weight:
@@ -485,8 +446,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
if not is_fusion_moe_shared_experts_layer:
loaded_params.add(name)
loaded_params.add(name)
loaded_layers: set[int] = set()
for param_name in loaded_params:
+44 -51
View File
@@ -16,12 +16,16 @@ from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -37,6 +41,9 @@ from vllm.model_executor.layers.mhc import (
MHCPreOp,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.config_utils import (
is_shared_expert_quant_fse_compatible,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
@@ -60,6 +67,8 @@ from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
logger = init_logger(__name__)
class DeepseekV4MLP(nn.Module):
def __init__(
@@ -154,48 +163,11 @@ class DeepseekV4MLP(nn.Module):
return x
def _shared_experts_are_fp4(config, layer_idx: int | None = None) -> bool:
"""Whether the shared experts are MXFP4 and thus fusable.
``layer_idx=None`` resolves the model-wide default (global scheme), used by
the main-model weight loader / mapper callers that operate per-model.
"""
quant_cfg = getattr(config, "quantization_config", None)
if quant_cfg is None:
return False
if layer_idx is None:
base = None
elif layer_idx >= config.num_hidden_layers:
base = f"mtp.{layer_idx - config.num_hidden_layers}.ffn.shared_experts"
else:
base = f"layers.{layer_idx}.ffn.shared_experts"
if base and any(e.startswith(base) for e in (quant_cfg.get("exclude") or [])):
return False
entry = (
(quant_cfg.get("layer_quant_config") or {}).get(f"{base}.w1") if base else None
)
if entry is None:
entry = quant_cfg.get("global_quant_config")
return ((entry or {}).get("weight") or {}).get("dtype") == "fp4"
def _fuse_shared_experts_enabled(config, prefix: str = "") -> bool:
"""Whether to fuse the shared expert into the routed MXFP4 grouped GEMM.
Fusion fuses the shared expert into the routed experts' MXFP4 grouped GEMM,
so it only applies where the shared expert is the same precision as the
routed experts. Some layers may carry a shared expert in a different quantization
than the routed experts; when so, it runs as its own linear and must not be fused.
"""
if not (
current_platform.is_rocm()
and getattr(config, "n_shared_experts", None)
def _fuse_shared_experts_enabled(config) -> bool:
return bool(
getattr(config, "n_shared_experts", None)
and envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
and not get_current_vllm_config().parallel_config.enable_expert_parallel
):
return False
return _shared_experts_are_fp4(
config, extract_layer_index(prefix) if prefix else None
)
@@ -255,9 +227,25 @@ class DeepseekV4MoE(nn.Module):
self.n_shared_experts = config.n_shared_experts
self.fuse_shared_experts = _fuse_shared_experts_enabled(config, prefix)
# TODO: Historically, only `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1`
# is checked to enable FSE for DeepSeek-v4, despite AITER not being used.
# This should be cleaned up and use `resolve_layer_fused_shared_expert`.
fse_requested = _fuse_shared_experts_enabled(config)
if fse_requested:
fse_compatible, fse_reason = is_shared_expert_quant_fse_compatible(
quant_config,
f"{prefix}.experts",
f"{prefix}.shared_experts",
)
if not fse_compatible:
logger.warning(
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
"cannot be enabled: %s.",
fse_reason,
)
self.is_fused_shared_expert_enabled = fse_requested and fse_compatible
if config.n_shared_experts is None or self.fuse_shared_experts:
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
@@ -282,8 +270,9 @@ class DeepseekV4MoE(nn.Module):
self.experts = FusedMoEFactory(
shared_experts=self.shared_experts,
n_shared_experts=(
config.n_shared_experts if self.fuse_shared_experts else None
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
),
fuse_shared_experts=self.is_fused_shared_expert_enabled,
gate=self.gate,
num_experts=config.n_routed_experts,
top_k=config.num_experts_per_tok,
@@ -581,6 +570,11 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
),
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
DeepseekV4MoE,
"ffn",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, self.rms_norm_eps)
@@ -763,7 +757,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
# diverge from how the module was built if per-layer quantization ever
# mixes fused and non-fused layers.
fuse_by_layer = {
extract_layer_index(mod_name): mod.fuse_shared_experts
extract_layer_index(mod_name): mod.is_fused_shared_expert_enabled
for mod_name, mod in self.named_modules()
if isinstance(mod, DeepseekV4MoE)
}
@@ -874,7 +868,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
# redirected shared-expert weights route through the expert loader.
n_shared = getattr(self.config, "n_shared_experts", 0) or 0
num_experts = self.config.n_routed_experts + (
n_shared if _fuse_shared_experts_enabled(self.config) else 0
n_shared if self.is_fused_shared_expert_enabled else 0
)
return fused_moe_make_expert_params_mapping(
self,
@@ -956,15 +950,14 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
config = vllm_config.model_config.hf_config
self.config = config
expert_dtype = getattr(config, "expert_dtype", "fp4")
fuse_shared_experts = _fuse_shared_experts_enabled(config)
if expert_dtype != "fp4" or fuse_shared_experts:
self.hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper(
expert_dtype, fuse_shared_experts=fuse_shared_experts
)
self.model = self.model_cls(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
if expert_dtype != "fp4" or self.model.is_fused_shared_expert_enabled:
self.hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper(
expert_dtype,
fuse_shared_experts=self.model.is_fused_shared_expert_enabled,
)
if get_pp_group().is_last_rank:
self.lm_head = ParallelLMHead(
config.vocab_size,
+64 -30
View File
@@ -34,6 +34,7 @@ from vllm.config import (
)
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention.attention import set_default_quant_scales
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
@@ -45,6 +46,9 @@ from vllm.model_executor.layers.fused_moe import (
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
MinimaxM3QKVParallelLinearWithIndexer,
@@ -53,6 +57,9 @@ from vllm.model_executor.layers.linear import (
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.config_utils import (
is_shared_expert_quant_fse_compatible,
)
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -108,23 +115,7 @@ from vllm.v1.kv_cache_interface import (
is_quantized_kv_cache,
)
def _fuse_shared_experts_enabled(config: PretrainedConfig) -> bool:
"""Whether to fuse the shared expert with routed experts.
ROCm only. Opt-in via ``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS`` (the
router-append fusion runs on both aiter and non-aiter MoE);
it is disabled under expert parallelism (the shared slot is appended to
the routed top-k, which the EP expert-mapping path does not handle).
"""
from vllm.platforms import current_platform
return bool(
current_platform.is_rocm()
and getattr(config, "n_shared_experts", None)
and envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
and not get_current_vllm_config().parallel_config.enable_expert_parallel
)
logger = init_logger(__name__)
def _sparse_attention_layer_ids(config: PretrainedConfig) -> set[int]:
@@ -303,10 +294,12 @@ class MiniMaxM3MLP(nn.Module):
return x
def _aiter_moe_fused_shared_experts_enabled(config: PretrainedConfig) -> bool:
def _aiter_moe_fused_shared_experts_enabled(
is_fused_shared_expert_enabled: bool,
) -> bool:
"""Whether the fused shared expert routes through aiter's grouped top-k MoE.
A strict sub-case of :func:`_fuse_shared_experts_enabled`: shared-expert
A strict sub-case of `is_fused_shared_expert_enabled`: shared-expert
fusion must already be opted in (``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS``)
and allowed (not under expert parallelism). When additionally on gfx950 with
an active aiter MoE backend, the shared expert is appended inside aiter's
@@ -314,11 +307,13 @@ def _aiter_moe_fused_shared_experts_enabled(config: PretrainedConfig) -> bool:
vLLM router's torch concat. Otherwise FSE still runs via the vLLM top-k bias
router.
"""
if not _fuse_shared_experts_enabled(config):
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950() and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
return (
on_gfx950()
and is_fused_shared_expert_enabled
and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
class MiniMaxM3MoE(nn.Module):
@@ -371,11 +366,46 @@ class MiniMaxM3MoE(nn.Module):
# MoE call as the last expert slot, so we don't build a separate module.
# On gfx950 with aiter MoE the append is fused inside aiter's grouped
# top-k kernel; otherwise it goes through the vLLM top-k bias router.
self.fuse_shared_experts = _fuse_shared_experts_enabled(config)
self.use_aiter_moe_fse = _aiter_moe_fused_shared_experts_enabled(config)
# It is disabled under expert parallelism (the shared slot is appended to
# the routed top-k, which the EP expert-mapping path does not handle).
# TODO: Historically, only `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1`
# is checked to enable FSE for MiniMax-M3, despite AITER not being used.
# This should be cleaned up and use `resolve_layer_fused_shared_expert`.
fse_requested = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
self.is_fused_shared_expert_enabled = False
if (
fse_requested
and bool(getattr(config, "n_shared_experts", None))
and not get_current_vllm_config().parallel_config.enable_expert_parallel
):
fse_compatible, fse_reason = is_shared_expert_quant_fse_compatible(
quant_config,
f"{prefix}.experts",
f"{prefix}.shared_experts",
)
if not fse_compatible:
logger.warning(
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
"cannot be enabled: %s.",
fse_reason,
)
else:
self.is_fused_shared_expert_enabled = True
# When additionally on gfx950 with an active aiter MoE backend, the shared
# expert is appended inside aiter's an active aiter MoE backend, the shared
# expert is appended inside aiter's biased grouped top-k kernel
# (``num_fused_shared_experts``) instead of the vLLM router's torch concat.
# Otherwise FSE still runs via the vLLM top-k bias router.
# TODO: `on_gfx950()` check here should not be MiniMax-M3 specific, and
# the check should be done on resolved MOE backend directly.
self.use_aiter_moe_fse = _aiter_moe_fused_shared_experts_enabled(
self.is_fused_shared_expert_enabled
)
self.shared_experts: MiniMaxM3MLP | None = None
if self.n_shared_experts and not self.fuse_shared_experts:
if self.n_shared_experts and not self.is_fused_shared_expert_enabled:
self.shared_experts = MiniMaxM3MLP(
config=config,
intermediate_size=config.intermediate_size * self.n_shared_experts,
@@ -412,8 +442,9 @@ class MiniMaxM3MoE(nn.Module):
router_logits_dtype=self.gate.out_dtype,
shared_experts=self.shared_experts,
n_shared_experts=(
self.n_shared_experts if self.fuse_shared_experts else None
self.n_shared_experts if self.is_fused_shared_expert_enabled else None
),
fuse_shared_experts=self.is_fused_shared_expert_enabled,
quant_config=quant_config,
prefix=f"{prefix}.experts",
)
@@ -1101,6 +1132,11 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
),
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
MiniMaxM3MoE,
"block_sparse_moe",
)
if get_pp_group().is_last_rank:
self.norm = MiniMAXGemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -1155,7 +1191,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
# expert, include the appended slot (id == num_local_experts).
n_shared = getattr(self.config, "n_shared_experts", 0) or 0
num_experts = self.config.num_local_experts + (
n_shared if _fuse_shared_experts_enabled(self.config) else 0
n_shared if self.is_fused_shared_expert_enabled else 0
)
return fused_moe_make_expert_params_mapping(
self,
@@ -1187,8 +1223,6 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
# (param_name, weight_name, expert_id, shard_id)
expert_params_mapping = self.get_expert_mapping()
_fuse_shared = _fuse_shared_experts_enabled(self.config)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
@@ -1206,7 +1240,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
# down->w2) so it loads via the routed expert loader. Runs before the
# stacked/dense mappings so shared_experts.gate_proj/up_proj are not
# captured by the dense gate_up_proj mapping.
if _fuse_shared and ".shared_experts." in name:
if self.is_fused_shared_expert_enabled and ".shared_experts." in name:
sid = self.config.num_local_experts
name = name.replace(".shared_experts.gate_proj.", f".experts.{sid}.w1.")
name = name.replace(".shared_experts.up_proj.", f".experts.{sid}.w3.")