diff --git a/tests/model_executor/layers/test_fused_shared_expert.py b/tests/model_executor/layers/test_fused_shared_expert.py new file mode 100644 index 0000000000..7080c66f12 --- /dev/null +++ b/tests/model_executor/layers/test_fused_shared_expert.py @@ -0,0 +1,700 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import importlib +from copy import deepcopy +from types import SimpleNamespace +from typing import Any, cast +from unittest.mock import patch + +import pytest +import torch +from torch import nn + +import vllm.config as vllm_config_module +from vllm.config import VllmConfig, set_current_vllm_config +from vllm.model_executor.layers.fused_moe import utils as fused_moe_utils +from vllm.model_executor.layers.fused_moe.layer import determine_expert_counts +from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig +from vllm.model_executor.layers.quantization.utils.config_utils import ( + is_shared_expert_quant_fse_compatible, +) +from vllm.model_executor.models.utils import PPMissingLayer +from vllm.models.deepseek_v4 import quant_config as deepseek_v4_quant_config +from vllm.models.minimax_m3.amd import model as minimax_m3_model +from vllm.transformers_utils.configs.minimax_m3 import MiniMaxM3TextConfig +from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig + +_QUARK_FSE_CONFIG: dict[str, Any] = { + "global_quant_config": { + "input_tensors": { + "dtype": "fp4", + "is_dynamic": True, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": None, + "symmetric": None, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", + "mx_element_dtype": None, + "observer_cls": "PerBlockMXObserver", + "is_scale_quant": False, + "enable_buffer_reuse": False, + "max_input_numel": 4194304, + }, + "output_tensors": None, + "weight": { + "dtype": "fp4", + "is_dynamic": False, + "qscheme": "per_group", + "ch_axis": -1, + "group_size": 32, + "block_size": None, + "symmetric": None, + "round_method": "half_even", + "scale_type": "float", + "scale_format": "e8m0", + "scale_calculation_mode": "even", + "mx_element_dtype": None, + "observer_cls": "PerBlockMXObserver", + "is_scale_quant": False, + "enable_buffer_reuse": False, + "max_input_numel": 4194304, + }, + "bias": None, + "target_device": None, + }, + "algo_config": None, + "softmax_quant_spec": None, + "quant_method": "quark", + "layer_type_quant_config": {}, + "layer_quant_config": {}, + "kv_cache_quant_config": {}, + "kv_cache_post_rope": False, + "quant_mode": "eager_mode", + "version": "0.12+9d3d471cdf1", + "export": { + "kv_cache_group": [], + "min_kv_scale": 0.0, + "pack_method": "reorder", + "weight_format": "real_quantized", + "weight_merge_groups": None, + }, +} + + +def get_deepseek_v4_quark_config(exclude: list[str]) -> dict[str, Any]: + """Return the DeepSeek-V4-Pro-MXFP4 FSE quantization layout.""" + # Mimics https://huggingface.co/amd/DeepSeek-V4-Pro-MXFP4. + quantization_config: dict[str, Any] = deepcopy(_QUARK_FSE_CONFIG) + quantization_config["exclude"] = exclude + mxfp4_config = cast( + dict[str, Any], deepcopy(quantization_config["global_quant_config"]) + ) + fp8_config = deepcopy(mxfp4_config) + fp8_config["input_tensors"].update( + { + "dtype": "fp8_e4m3", + "group_size": 128, + "symmetric": True, + "scale_type": None, + "scale_format": None, + "scale_calculation_mode": None, + "observer_cls": "PerGroupMinMaxObserver", + } + ) + fp8_config["weight"].update( + { + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "ch_axis": None, + "group_size": None, + "block_size": [128, 128], + "symmetric": True, + "scale_type": "float8_e8m0fnu", + "scale_format": None, + "scale_calculation_mode": None, + "observer_cls": "PerBlock2DMinMaxObserver", + } + ) + quantization_config["layer_quant_config"] = { + r"re:mtp\.0\.ffn\.experts\.\d+\.w1": mxfp4_config, + r"re:mtp\.0\.ffn\.shared_experts\.w1": fp8_config, + } + return quantization_config + + +def get_fse_test_model_config( + model_type: str, + quantization_config: dict[str, Any], +) -> tuple[object, type[nn.Module]]: + if model_type == "minimax_m3": + config = MiniMaxM3TextConfig( + hidden_size=128, + intermediate_size=32, + dense_intermediate_size=32, + num_hidden_layers=2, + num_attention_heads=1, + num_key_value_heads=1, + head_dim=128, + num_local_experts=2, + num_experts_per_tok=1, + moe_layer_freq=[0, 1], + sparse_attention_config=None, + rotary_dim=64, + quantization_config=quantization_config, + ) + return config, minimax_m3_model.MiniMaxM3Model + if model_type == "deepseek_v4": + from vllm.models.deepseek_v4.amd.model import DeepseekV4Model + + config = SimpleNamespace( + vocab_size=256, + hidden_size=128, + num_hidden_layers=1, + num_attention_heads=1, + head_dim=128, + max_position_embeddings=128, + sliding_window=None, + compress_ratios=[1], + rope_theta=10000.0, + compress_rope_theta=10000.0, + rope_parameters={"rope_type": "default"}, + rms_norm_eps=1e-6, + hidden_act="silu", + q_lora_rank=0, + o_lora_rank=0, + o_groups=1, + qk_rope_head_dim=64, + index_head_dim=64, + index_n_heads=1, + index_topk=1, + n_routed_experts=2, + n_shared_experts=1, + num_experts_per_tok=1, + n_group=1, + topk_group=1, + moe_intermediate_size=32, + norm_topk_prob=False, + num_hash_layers=0, + swiglu_limit=0.0, + hc_eps=1e-6, + hc_mult=1, + hc_sinkhorn_iters=1, + expert_dtype="fp4", + num_nextn_predict_layers=1, + quantization_config=quantization_config, + ) + return config, DeepseekV4Model + if model_type == "qwen3_5": + from vllm.model_executor.models.qwen3_5 import Qwen3_5Model + + config = Qwen3_5MoeTextConfig( + vocab_size=256, + hidden_size=128, + num_hidden_layers=1, + num_attention_heads=1, + num_key_value_heads=1, + head_dim=128, + linear_key_head_dim=64, + linear_value_head_dim=64, + linear_num_key_heads=1, + linear_num_value_heads=1, + moe_intermediate_size=32, + shared_expert_intermediate_size=32, + num_experts_per_tok=1, + num_experts=2, + layer_types=["full_attention"], + quantization_config=quantization_config, + ) + return config, Qwen3_5Model + if model_type == "deepseek_v2": + from transformers import DeepseekV2Config + + from vllm.model_executor.models.deepseek_v2 import DeepseekV2Model + + config = DeepseekV2Config( + vocab_size=256, + hidden_size=128, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=1, + num_key_value_heads=1, + head_dim=128, + first_k_dense_replace=0, + n_routed_experts=2, + n_shared_experts=1, + num_experts_per_tok=1, + n_group=1, + topk_group=1, + moe_intermediate_size=32, + q_lora_rank=None, + kv_lora_rank=0, + qk_nope_head_dim=0, + qk_rope_head_dim=0, + v_head_dim=0, + quantization_config=quantization_config, + ) + return config, DeepseekV2Model + if model_type == "glm4_moe": + from transformers.models.glm4_moe import Glm4MoeConfig + + from vllm.model_executor.models.glm4_moe import Glm4MoeModel + + config = Glm4MoeConfig( + vocab_size=256, + hidden_size=128, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=1, + num_key_value_heads=1, + head_dim=128, + moe_intermediate_size=32, + n_routed_experts=2, + n_shared_experts=1, + num_experts_per_tok=1, + first_k_dense_replace=0, + quantization_config=quantization_config, + ) + return config, Glm4MoeModel + raise ValueError(f"Unsupported FSE test model: {model_type}") + + +def test_determine_expert_counts_fuse_shared_experts_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + common_args = (8, 0, 2) + assert determine_expert_counts(*common_args, True)[2] == 2 + assert determine_expert_counts(*common_args, False)[2] == 0 + + +def test_resolve_layer_fused_shared_expert_skips_compatibility_when_disabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + fused_moe_utils.rocm_aiter_ops, + "is_fusion_moe_shared_experts_enabled", + lambda: False, + ) + monkeypatch.setattr( + fused_moe_utils, + "is_shared_expert_quant_fse_compatible", + lambda *_: pytest.fail( + "compatibility must not be checked when FSE is disabled" + ), + ) + + assert not fused_moe_utils.resolve_layer_fused_shared_expert( + object(), "model.layers.0.mlp" + ) + + +def test_resolve_layer_fused_shared_expert_normalizes_unavailable_aiter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + fused_moe_utils.rocm_aiter_ops, + "is_fusion_moe_shared_experts_enabled", + lambda: None, + ) + + assert ( + fused_moe_utils.resolve_layer_fused_shared_expert( + object(), "model.layers.0.mlp" + ) + is False + ) + + +def test_resolve_layer_fused_shared_expert_passes_module_prefixes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + quant_config = object() + monkeypatch.setattr( + fused_moe_utils.rocm_aiter_ops, + "is_fusion_moe_shared_experts_enabled", + lambda: True, + ) + + calls: list[tuple[object, str, str]] = [] + + def check_compatibility( + config: object, expert_prefix: str, shared_expert_prefix: str + ) -> tuple[bool, str | None]: + calls.append((config, expert_prefix, shared_expert_prefix)) + return True, None + + monkeypatch.setattr( + fused_moe_utils, "is_shared_expert_quant_fse_compatible", check_compatibility + ) + + assert fused_moe_utils.resolve_layer_fused_shared_expert( + quant_config, + "model.layers.0.mlp", + shared_expert_name="shared_expert", + ) + assert calls == [ + ( + quant_config, + "model.layers.0.mlp.experts", + "model.layers.0.mlp.shared_expert", + ) + ] + + +def test_resolve_layer_fused_shared_expert_rejects_incompatible_quantization( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setattr( + fused_moe_utils.rocm_aiter_ops, + "is_fusion_moe_shared_experts_enabled", + lambda: True, + ) + monkeypatch.setattr( + fused_moe_utils, + "is_shared_expert_quant_fse_compatible", + lambda *_: (False, "shared experts are excluded"), + ) + + assert not fused_moe_utils.resolve_layer_fused_shared_expert( + object(), "model.layers.0.mlp" + ) + assert "shared experts are excluded" in caplog.text + + +def test_deepseek_v4_shared_expert_fse_uses_mtp_quantization_config_prefix( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class DeepseekV4Config: + expert_dtype = "fp4" + + def _is_quark_mxfp4_ocp(self, hf_config: object) -> bool: + return True + + hf_config = SimpleNamespace( + num_hidden_layers=2, + quantization_config={ + "layer_quant_config": { + r"re:mtp\.0\.ffn\.shared_experts\.w1": {"weight": {"dtype": "fp4"}} + }, + "global_quant_config": {"weight": {"dtype": "fp8"}}, + }, + ) + monkeypatch.setattr( + deepseek_v4_quant_config, "DeepseekV4FP8Config", DeepseekV4Config + ) + monkeypatch.setattr( + vllm_config_module, + "get_current_vllm_config", + lambda: SimpleNamespace(model_config=SimpleNamespace(hf_config=hf_config)), + ) + + compatible, reason = is_shared_expert_quant_fse_compatible( + DeepseekV4Config(), + "model.layers.2.ffn.experts", + "model.layers.2.ffn.shared_experts", + ) + + assert compatible + assert reason is None + + +def test_is_model_fused_shared_expert_compatible() -> None: + class MoE(nn.Module): + def __init__(self, enabled: bool) -> None: + super().__init__() + self.is_fused_shared_expert_enabled = enabled + + class Layer(nn.Module): + def __init__(self, enabled: bool) -> None: + super().__init__() + self.mlp = MoE(enabled) + + enabled_layers = nn.ModuleList([Layer(True)]) + disabled_layers = nn.ModuleList([Layer(False)]) + mixed_layers = nn.ModuleList([Layer(True), Layer(False)]) + empty_layers = nn.ModuleList() + pipeline_layers = nn.ModuleList([Layer(True), PPMissingLayer()]) + + assert fused_moe_utils.is_model_fused_shared_expert_compatible( + enabled_layers, MoE, "mlp" + ) + assert not fused_moe_utils.is_model_fused_shared_expert_compatible( + disabled_layers, MoE, "mlp" + ) + assert not fused_moe_utils.is_model_fused_shared_expert_compatible( + empty_layers, MoE, "mlp" + ) + assert fused_moe_utils.is_model_fused_shared_expert_compatible( + pipeline_layers, MoE, "mlp" + ) + with pytest.raises(NotImplementedError, match="1 enabled and 1 disabled layers"): + fused_moe_utils.is_model_fused_shared_expert_compatible( + mixed_layers, MoE, "mlp" + ) + + +@pytest.mark.parametrize( + "model_type", + ["minimax_m3", "deepseek_v4", "qwen3_5", "glm4_moe", "deepseek_v2"], +) +@pytest.mark.parametrize( + ("use_fse", "exclude"), + [ + (False, []), + (True, []), + (True, ["*.shared_experts.*"]), + ], +) +def test_models_fse_init( + model_type: str, + use_fse: bool, + exclude: list[str], + dist_init: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Model construction resolves FSE consistently with Quark quantization.""" + + quantization_config: dict[str, Any] = ( + get_deepseek_v4_quark_config(["layers.0.ffn.shared_experts"] if exclude else []) + if model_type == "deepseek_v4" + else {**_QUARK_FSE_CONFIG, "exclude": exclude} + ) + + config, model_constructor = get_fse_test_model_config( + model_type, quantization_config + ) + vllm_config = VllmConfig() + vllm_config.model_config = SimpleNamespace( + hf_config=config, + hf_text_config=config, + dtype=torch.bfloat16, + max_model_len=128, + is_diffusion=False, + head_dtype=None, + is_mm_prefix_lm=False, + multimodal_config=None, + quantization_config=None, + ) + vllm_config.parallel_config.enable_expert_parallel = False + if model_type == "deepseek_v4": + from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config + + vllm_config.cache_config.cache_dtype = "fp8_ds_mla" + vllm_config.quant_config = DeepseekV4FP8Config( + is_checkpoint_fp8_serialized=True, + weight_block_size=[128, 128], + ) + else: + vllm_config.quant_config = QuarkConfig(quantization_config) + + import vllm.envs as envs + from vllm._aiter_ops import rocm_aiter_ops + + if model_type == "deepseek_v4": + from vllm.models.deepseek_v4.amd import model as deepseek_v4_model + + warning_logger = deepseek_v4_model.logger + elif model_type == "minimax_m3": + warning_logger = minimax_m3_model.logger + else: + warning_logger = fused_moe_utils.logger + + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_USE_AITER", str(use_fse)) + mp.setenv("VLLM_ROCM_USE_AITER_MOE", str(use_fse)) + mp.setenv("VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", str(use_fse)) + mp.setattr( + "vllm.model_executor.layers.fused_moe.experts." + "ocp_mx_emulation_moe.has_quark", + lambda: True, + ) + importlib.reload(envs) + mp.setattr(envs, "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", use_fse) + rocm_aiter_ops.refresh_env_variables() + aiter_fse_enabled = bool(rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()) + # These AMD-specific models currently use the raw environment flag. + # and do not rely on `rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()`. + fse_enabled = use_fse and ( + aiter_fse_enabled or model_type in {"deepseek_v4", "minimax_m3"} + ) + + with ( + patch.object(warning_logger, "warning") as warning, + set_current_vllm_config(vllm_config), + ): + model = model_constructor(vllm_config=vllm_config) + mtp = None + if model_type == "deepseek_v4" and use_fse and not exclude: + from vllm.models.deepseek_v4.amd.mtp import ( + DeepSeekV4MTP, + ) + + vllm_config.speculative_config = SimpleNamespace( + draft_model_config=SimpleNamespace(hf_config=config) + ) + mtp = DeepSeekV4MTP(vllm_config=vllm_config) + assert model.is_fused_shared_expert_enabled is (fse_enabled and not exclude) + + # The dummy quant config here uses mixed mxfp4/fp8 for experts/shared_expert + # so should just raise a warning. + if mtp is not None: + assert not mtp.model.layers[ + "1" + ].mtp_block.ffn.is_fused_shared_expert_enabled + warning.assert_called_once() + assert ( + "DeepSeek-V4 shared experts at mtp.0.ffn.shared_experts" + in (warning.call_args.args[1]) + ) + if aiter_fse_enabled and exclude: + warning.assert_called_once() + assert ( + "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled" + in (warning.call_args.args[0]) + ) + assert "excludes shared experts" in warning.call_args.args[1] + + importlib.reload(envs) + rocm_aiter_ops.refresh_env_variables() + + +@pytest.mark.parametrize( + ("exclude", "expected"), + [([], True), (["*.shared_expert.*"], False)], +) +def test_quark_shared_expert_fse_compatibility( + exclude: list[str], expected: bool +) -> None: + compatible, reason = is_shared_expert_quant_fse_compatible( + QuarkConfig( + { + "exclude": exclude, + "global_quant_config": {}, + "layer_quant_config": {}, + } + ), + "model.layers.0.mlp.experts", + "model.layers.0.mlp.shared_expert", + ) + + assert compatible is expected + if expected: + assert reason is None + else: + assert ( + reason + == "Quark excludes shared experts at model.layers.0.mlp.shared_expert" + ) + + +def test_quark_shared_expert_fse_requires_matching_layer_quant_configs() -> None: + global_quant_config = {"weight": {"dtype": "fp4"}} + quant_config = QuarkConfig( + { + "exclude": [], + "global_quant_config": global_quant_config, + "layer_quant_config": { + "model.layers.0.mlp.shared_expert.gate_up_proj": { + "weight": {"dtype": "fp8"} + }, + "model.layers.0.mlp.shared_expert.down_proj": { + "weight": {"dtype": "fp8"} + }, + }, + } + ) + + compatible, reason = is_shared_expert_quant_fse_compatible( + quant_config, + "model.layers.0.mlp.experts", + "model.layers.0.mlp.shared_expert", + ) + + assert not compatible + assert reason == ( + "Quark uses different quantization configurations for routed and " + "shared experts at model.layers.0.mlp.shared_expert" + ) + + +def test_quark_shared_expert_fse_rejects_partial_packed_projection_override() -> None: + fp4_config = {"weight": {"dtype": "fp4"}} + fp8_config = {"weight": {"dtype": "fp8"}} + quant_config = QuarkConfig( + { + "exclude": [], + "global_quant_config": fp4_config, + "layer_quant_config": { + "model.layers.0.mlp.experts": fp8_config, + "model.layers.0.mlp.shared_expert.gate_proj": fp8_config, + "model.layers.0.mlp.shared_expert.down_proj": fp8_config, + }, + } + ) + quant_config.packed_modules_mapping = {"gate_up_proj": ["gate_proj", "up_proj"]} + + compatible, reason = is_shared_expert_quant_fse_compatible( + quant_config, + "model.layers.0.mlp.experts", + "model.layers.0.mlp.shared_expert", + ) + + assert not compatible + assert reason == ( + "Quark uses different quantization configurations for routed and " + "shared experts at model.layers.0.mlp.shared_expert" + ) + + +def test_quark_layer_config_from_name_checks_packed_projections() -> None: + quant_config = QuarkConfig( + { + "global_quant_config": {"weight": {"dtype": "fp4"}}, + "layer_quant_config": { + "model.layers.0.mlp.shared_expert.w1": {"weight": {"dtype": "fp4"}}, + "model.layers.0.mlp.shared_expert.w3": {"weight": {"dtype": "fp8"}}, + }, + } + ) + quant_config.packed_modules_mapping = {"gate_up_proj": ["w1", "w3"]} + + with pytest.raises(ValueError, match="requires all to use the same scheme"): + quant_config.get_layer_quant_config_from_name( + "model.layers.0.mlp.shared_expert.gate_up_proj" + ) + + +def test_quark_packed_layer_config_must_match_global_config() -> None: + quant_config = QuarkConfig( + { + "global_quant_config": {"weight": {"dtype": "fp4"}}, + "layer_type_quant_config": {}, + "layer_quant_config": { + "model.layers.0.mlp.shared_expert.w1": {"weight": {"dtype": "fp8"}}, + }, + } + ) + quant_config.packed_modules_mapping = {"gate_up_proj": ["w1", "w3"]} + + with pytest.raises(ValueError, match="requires all to use the same scheme"): + quant_config._find_matched_config( + "model.layers.0.mlp.shared_expert.gate_up_proj", nn.Module() + ) + + +def test_non_quark_shared_expert_fse_is_incompatible() -> None: + compatible, reason = is_shared_expert_quant_fse_compatible( + object(), + "model.layers.0.mlp.experts", + "model.layers.0.mlp.shared_experts", + ) + + assert not compatible + assert reason == ( + "shared-expert FSE quantization compatibility is not implemented for object" + ) diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 28b994057f..d9fd1385a9 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -6,7 +6,6 @@ from typing import Any import torch -import vllm.envs as envs from vllm._aiter_ops import rocm_aiter_ops from vllm.config import ParallelConfig, get_current_vllm_config from vllm.distributed import ( @@ -74,23 +73,13 @@ def determine_expert_counts( num_experts: int, num_redundant_experts: int, n_shared_experts: int | None, - is_act_and_mul: bool, + fuse_shared_experts: bool, ) -> tuple[int, int, int]: global_num_experts = num_experts + num_redundant_experts logical_num_experts = num_experts - # Shared-expert fusion: append the shared expert(s) as routed-expert slots - # so they run in the same grouped GEMM. Gated by - # VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: either the native aiter fused-MoE - # path (env + master switch, via is_fusion_moe_shared_experts_enabled) or the - # backend-neutral router-append path (env alone, independent of the master - # switch; e.g. the MM3 triton/flydsl mxfp8 MoE). Gated activations only. - fuse_shared_enabled = ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - or envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS - ) and is_act_and_mul num_fused_shared_experts = ( - n_shared_experts if n_shared_experts is not None and fuse_shared_enabled else 0 + n_shared_experts if n_shared_experts is not None and fuse_shared_experts else 0 ) return global_num_experts, logical_num_experts, num_fused_shared_experts @@ -132,6 +121,7 @@ def FusedMoEFactory( ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"), is_fused_checkpoint_transposed: bool = False, n_shared_experts: int | None = None, + fuse_shared_experts: bool = False, router_logits_dtype: torch.dtype | None = None, gate: torch.nn.Module | None = None, shared_experts: torch.nn.Module | None = None, @@ -198,6 +188,7 @@ def FusedMoEFactory( block scales use transposed storage. n_shared_experts: Number of shared experts to fuse into the routed grouped GEMM (ROCm; requires aiter FSE or the router-append path) + fuse_shared_experts: Whether to enable shared-expert fusion. router_logits_dtype: Data type for router logits buffers gate: Pre-configured gate module shared_experts: Pre-configured shared experts module @@ -244,7 +235,7 @@ def FusedMoEFactory( num_experts, num_redundant_experts, n_shared_experts, - is_act_and_mul, + fuse_shared_experts, ) ) diff --git a/vllm/model_executor/layers/fused_moe/utils.py b/vllm/model_executor/layers/fused_moe/utils.py index cce8ccd073..b873d8321d 100644 --- a/vllm/model_executor/layers/fused_moe/utils.py +++ b/vllm/model_executor/layers/fused_moe/utils.py @@ -1,15 +1,21 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import functools +from collections.abc import Iterable from math import prod from typing import TYPE_CHECKING import torch import torch.nn.functional as F +from torch import nn import vllm.envs as envs from vllm import _custom_ops as ops +from vllm._aiter_ops import rocm_aiter_ops from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.utils.config_utils import ( + is_shared_expert_quant_fse_compatible, +) from vllm.model_executor.layers.quantization.utils.fp8_utils import ( per_token_group_quant_fp8, ) @@ -34,16 +40,95 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( per_tensor_dequantize, ) +from vllm.model_executor.models.utils import PPMissingLayer from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.math_utils import cdiv if TYPE_CHECKING: from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig + from vllm.model_executor.layers.quantization import QuantizationConfig logger = init_logger(__name__) +def resolve_layer_fused_shared_expert( + quant_config: "QuantizationConfig | None", + prefix: str, + shared_expert_name: str = "shared_experts", +) -> bool: + """Resolve whether AITER fused shared-expert execution is enabled. + + Args: + quant_config: Model quantization configuration. + prefix: MoE module prefix. + shared_expert_name: Shared-expert module name under ``prefix``. + + Returns: + Whether AITER fused shared experts are enabled. + + Raises: + ValueError: If requested shared-expert fusion is quantization-incompatible. + """ + # NOTE: is_fusion_moe_shared_experts_enabled is decorated with @if_aiter_supported + # that returns None if AITER is not available. + fse_requested = bool(rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()) + fse_compatible, fse_reason = ( + is_shared_expert_quant_fse_compatible( + quant_config, + f"{prefix}.experts", + f"{prefix}.{shared_expert_name}", + ) + if fse_requested + else (True, None) + ) + is_fused_shared_expert_enabled = fse_requested and fse_compatible + if fse_requested and not is_fused_shared_expert_enabled: + logger.warning( + "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but " + "cannot be enabled: %s.", + fse_reason, + ) + return is_fused_shared_expert_enabled + + +def is_model_fused_shared_expert_compatible( + layers: nn.ModuleList | Iterable[nn.Module], + moe_cls: type[nn.Module], + moe_name: str, +) -> bool: + """Resolve one fused-shared-expert state for a model's MoE layers.""" + + def get_moe_layer(layer: nn.Module) -> nn.Module | None: + for name in moe_name.split("."): + layer = getattr(layer, name, None) + if layer is None: + return None + return layer + + moe_layers = ( + moe_layer + for layer in layers + if not isinstance(layer, PPMissingLayer) + and (moe_layer := get_moe_layer(layer)) is not None + and isinstance(moe_layer, moe_cls) + ) + + enabled = [ + getattr(layer, "is_fused_shared_expert_enabled", False) for layer in moe_layers + ] + enabled_count = sum(enabled) + disabled_count = len(enabled) - enabled_count + if enabled_count > 0 and disabled_count > 0: + raise NotImplementedError( + "Fused shared experts must be enabled for all MoE layers; found " + f"{enabled_count} enabled and {disabled_count} disabled layers. " + "Per-layer fused shared experts is not yet supported. Please open " + "an issue." + ) + return enabled_count > 0 and disabled_count == 0 + + @triton.jit def _count_expert_num_tokens( topk_ids_ptr, diff --git a/vllm/model_executor/layers/quantization/quark/quark.py b/vllm/model_executor/layers/quantization/quark/quark.py index 9a65432c5c..912f61ac38 100644 --- a/vllm/model_executor/layers/quantization/quark/quark.py +++ b/vllm/model_executor/layers/quantization/quark/quark.py @@ -576,63 +576,91 @@ class QuarkConfig(QuantizationConfig): return True - def _find_matched_config( - self, layer_name: str, module: torch.nn.Module - ) -> dict[str, Any]: + def get_layer_quant_config_from_name( + self, layer_name: str + ) -> dict[str, Any] | None: proj_name = layer_name.split(".")[-1] if proj_name in self.packed_modules_mapping: shard_proj_names = self.packed_modules_mapping[proj_name] - - # Convert fused_name --> [shard_names] - shard_names = [ - layer_name.replace(proj_name, shard_proj_name) - for shard_proj_name in shard_proj_names - ] - shard_configs = [] - for shard_name in shard_names: - if shard_name == layer_name: - config = cast( - dict[str, Any], self.quant_config.get("global_quant_config") - ) + for shard_proj_name in shard_proj_names: + shard_name = layer_name.replace(proj_name, shard_proj_name) + if shard_name != layer_name: + config = self.get_layer_quant_config_from_name(shard_name) else: - config = self._find_matched_config(shard_name, module) + config = None shard_configs.append(config) - if not all( - deep_compare(q_config, shard_configs[0]) for q_config in shard_configs + matched_configs = [config for config in shard_configs if config is not None] + if matched_configs and not all( + deep_compare(config, matched_configs[0]) for config in matched_configs ): raise ValueError( f"Found a different quantization configuration for " f"{shard_proj_names} in {layer_name}. vLLM " "requires all to use the same scheme." ) - return shard_configs[0] + if matched_configs: + return matched_configs[0] + return None else: layer_quant_config = cast( - dict[str, Any], self.quant_config.get("layer_quant_config") + dict[str, Any], self.quant_config.get("layer_quant_config") or {} ) - - def _matches_pattern(layer_name, pattern): - if "*" not in pattern: - return layer_name in pattern - return fnmatch.fnmatch(layer_name, pattern) - for name_pattern, config in layer_quant_config.items(): - if _matches_pattern(layer_name, name_pattern): + if "*" not in name_pattern: + matches = layer_name in name_pattern + else: + matches = fnmatch.fnmatch(layer_name, name_pattern) + if matches: return config + return None - layer_type = cast(str, type(module)) - layer_type_quant_config = cast( - dict[str, Any], self.quant_config.get("layer_type_quant_config") - ) - if layer_type in layer_type_quant_config: - return layer_type_quant_config[layer_type] + def _find_matched_config( + self, layer_name: str, module: torch.nn.Module + ) -> dict[str, Any]: + # Priority order: + # 1. layer_quant_config, + # 2. layer_type_quant_config, + # 3. global_quant_config. - global_quant_config = cast( - dict[str, Any], self.quant_config.get("global_quant_config") - ) - return global_quant_config + layer_type = cast(str, type(module)) + layer_type_quant_config = cast( + dict[str, Any], self.quant_config.get("layer_type_quant_config") + ) + global_quant_config = cast( + dict[str, Any], self.quant_config.get("global_quant_config") + ) + fallback_config = layer_type_quant_config.get(layer_type, global_quant_config) + + proj_name = layer_name.split(".")[-1] + if proj_name in self.packed_modules_mapping: + shard_proj_names = self.packed_modules_mapping[proj_name] + shard_configs = [] + for shard_proj_name in shard_proj_names: + shard_name = layer_name.replace(proj_name, shard_proj_name) + if shard_name == layer_name: + config = fallback_config + else: + config = self.get_layer_quant_config_from_name(shard_name) + if config is None: + config = fallback_config + shard_configs.append(config) + + if not all( + deep_compare(config, shard_configs[0]) for config in shard_configs + ): + raise ValueError( + f"Found a different quantization configuration for " + f"{shard_proj_names} in {layer_name}. vLLM requires all " + "to use the same scheme." + ) + return shard_configs[0] + else: + layer_quant_config = self.get_layer_quant_config_from_name(layer_name) + if layer_quant_config is not None: + return layer_quant_config + return fallback_config def _get_scheme_from_config( self, config: dict[str, Any], dynamic_mxfp4_quant: bool = False diff --git a/vllm/model_executor/layers/quantization/utils/config_utils.py b/vllm/model_executor/layers/quantization/utils/config_utils.py new file mode 100644 index 0000000000..00cdd0c87d --- /dev/null +++ b/vllm/model_executor/layers/quantization/utils/config_utils.py @@ -0,0 +1,155 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import TYPE_CHECKING + +import regex as re + +if TYPE_CHECKING: + from vllm.model_executor.layers.quantization.base_config import QuantizationConfig + + +def is_shared_expert_quant_fse_compatible( + quant_config: "QuantizationConfig | None", + expert_prefix: str, + shared_expert_prefix: str, + projection_names: list[str] | None = None, +) -> tuple[bool, str | None]: + """Check whether quantization permits fused shared-expert execution. + + Args: + quant_config: Model quantization configuration. + expert_prefix: Routed-expert module prefix. + shared_expert_prefix: Shared-expert module prefix. + projection_names: Shared-expert projection names. + + Returns: + A compatibility flag and, when incompatible, the reason. + """ + if projection_names is None: + projection_names = ["gate_up_proj", "down_proj"] + + if quant_config is None: + return True, None + + from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig + from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config + + if isinstance(quant_config, DeepseekV4FP8Config): + from vllm.config import get_current_vllm_config + from vllm.model_executor.models.utils import extract_layer_index + + if quant_config.expert_dtype != "fp4": + return False, "DeepSeek-V4 routed experts are not MXFP4" + + hf_config = get_current_vllm_config().model_config.hf_config + + # TODO: This is adapted from former `_shared_experts_are_fp4`, and + # needs to be cleaned up this . There should not be Quark-specific + # logic in DeepseekV4FP8Config. + quantization_config = getattr(hf_config, "quantization_config", None) + if quantization_config is None: + return False, "DeepSeek-V4 has no quantization configuration" + + if not quant_config._is_quark_mxfp4_ocp(quantization_config): + return False, "DeepSeek-v4 FSE is only implemented/tested with Quark MXFP4" + + layer_idx = extract_layer_index(shared_expert_prefix) + if layer_idx >= hf_config.num_hidden_layers: + shared_expert_prefix = ( + f"mtp.{layer_idx - hf_config.num_hidden_layers}.ffn.shared_experts" + ) + else: + shared_expert_prefix = f"layers.{layer_idx}.ffn.shared_experts" + + if any( + entry.startswith(shared_expert_prefix) + for entry in quantization_config.get("exclude") or [] + if isinstance(entry, str) + ): + return ( + False, + f"DeepSeek-V4 excludes shared experts at {shared_expert_prefix}", + ) + + shared_expert_weight_name = f"{shared_expert_prefix}.w1" + layer_quant_config = quantization_config.get("layer_quant_config") or {} + layer_config = layer_quant_config.get(shared_expert_weight_name) + if layer_config is None: + layer_config = next( + ( + config + for pattern, config in layer_quant_config.items() + if isinstance(pattern, str) + and pattern.startswith("re:") + and re.fullmatch( + pattern.removeprefix("re:"), shared_expert_weight_name + ) + ), + None, + ) + shared_weight_config = ( + layer_config or quantization_config.get("global_quant_config") or {} + ).get("weight") or {} + if shared_weight_config.get("dtype") == "fp4": + return True, None + return ( + False, + f"DeepSeek-V4 shared experts at {shared_expert_prefix} are not MXFP4", + ) + + if isinstance(quant_config, QuarkConfig): + # TODO: layer_type_quant_config is not taken into account here. + assert "exclude" in quant_config.quant_config + assert "global_quant_config" in quant_config.quant_config + + is_compatible = not any( + "shared_expert" in str(entry) + for entry in quant_config.quant_config["exclude"] + ) + if not is_compatible: + return False, f"Quark excludes shared experts at {shared_expert_prefix}" + + global_quant_config = quant_config.quant_config["global_quant_config"] + + def get_projection_quant_configs(layer_name: str) -> list[object]: + module_prefix, _, projection_name = layer_name.rpartition(".") + packed_projection_names = quant_config.packed_modules_mapping.get( + projection_name, [projection_name] + ) + return [ + quant_config.get_layer_quant_config_from_name( + f"{module_prefix}.{packed_projection_name}" + ) + or global_quant_config + for packed_projection_name in packed_projection_names + ] + + expert_quant_config = ( + quant_config.get_layer_quant_config_from_name(expert_prefix) + or global_quant_config + ) + shared_expert_quant_configs = [ + config + for projection_name in projection_names + for config in get_projection_quant_configs( + f"{shared_expert_prefix}.{projection_name}" + ) + ] + if all(config == expert_quant_config for config in shared_expert_quant_configs): + return True, None + return ( + False, + "Quark uses different quantization configurations for routed and " + f"shared experts at {shared_expert_prefix}", + ) + + # TODO: Extend FSE support detection to other quantization methods. Typically, + # one would check that the experts and shared_experts use the same + # quantization config. This may be refactored as part of QuantizationConfig later. + + return ( + False, + "shared-expert FSE quantization compatibility is not implemented for " + f"{type(quant_config).__name__}", + ) diff --git a/vllm/model_executor/models/AXK1.py b/vllm/model_executor/models/AXK1.py index 6000e62828..031808ee12 100644 --- a/vllm/model_executor/models/AXK1.py +++ b/vllm/model_executor/models/AXK1.py @@ -46,6 +46,10 @@ from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, + resolve_layer_fused_shared_expert, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -148,10 +152,14 @@ class AXK1MoE(nn.Module): self.n_local_physical_experts = self.n_physical_experts // self.ep_size self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled() - self.is_fusion_moe_shared_experts_enabled = ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - ) - if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled: + + self.is_fused_shared_expert_enabled = False + if config.n_shared_experts is not None: + self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert( + quant_config, prefix + ) + + if config.n_shared_experts is None or self.is_fused_shared_expert_enabled: self.shared_experts = None else: intermediate_size = config.moe_intermediate_size * config.n_shared_experts @@ -189,8 +197,9 @@ class AXK1MoE(nn.Module): num_redundant_experts=self.n_redundant_experts, is_sequence_parallel=self.is_sequence_parallel, n_shared_experts=config.n_shared_experts - if self.is_fusion_moe_shared_experts_enabled + if self.is_fused_shared_expert_enabled else None, + fuse_shared_experts=self.is_fused_shared_expert_enabled, ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -725,6 +734,12 @@ class AXK1Model(nn.Module): prefix=f"{prefix}.layers", ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + AXK1MoE, + "mlp", + ) + if get_pp_group().is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) else: @@ -789,9 +804,6 @@ class AXK1Model(nn.Module): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - rocm_aiter_moe_shared_expert_enabled = ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - ) stacked_params_mapping: list[tuple[str, str, int | str]] = [ # (param_name, shard_name, shard_id) ("gate_up_proj", "gate_proj", 0), @@ -822,7 +834,7 @@ class AXK1Model(nn.Module): num_experts=self.config.n_routed_experts + ( (self.config.n_shared_experts or 0) - if rocm_aiter_moe_shared_expert_enabled + if self.is_fused_shared_expert_enabled else 0 ), num_redundant_experts=self.num_redundant_experts, @@ -839,7 +851,7 @@ class AXK1Model(nn.Module): continue # skip spec decode layers for main model is_fusion_moe_shared_experts_layer = ( - rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name) + self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name) ) for param_name, weight_name, shard_id in stacked_params_mapping: diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py index 7fc78f13ff..8ac43e0bb2 100644 --- a/vllm/model_executor/models/deepseek_mtp.py +++ b/vllm/model_executor/models/deepseek_mtp.py @@ -7,13 +7,15 @@ import torch import torch.nn as nn from transformers import PretrainedConfig -from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed import tensor_model_parallel_all_gather from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig @@ -255,6 +257,11 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts): self.moe_mlp_layers.append(layer.mlp) self.moe_layers.append(layer.mlp.experts) self.extract_moe_parameters(example_moe) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.model.layers.values(), + DeepseekV2MoE, + "mtp_block.mlp", + ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.model.embed_input_ids(input_ids) @@ -285,9 +292,6 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts): return self.model.compute_logits(hidden_states, spec_step_idx) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - rocm_aiter_moe_shared_expert_enabled = ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - ) stacked_params_mapping = [ ("gate_up_proj", "gate_proj", 0), ("gate_up_proj", "up_proj", 1), @@ -310,7 +314,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts): num_experts=self.config.n_routed_experts + ( self.config.n_shared_experts - if rocm_aiter_moe_shared_expert_enabled + if self.is_fused_shared_expert_enabled else 0 ), ) @@ -326,7 +330,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts): if spec_layer is None: continue is_fusion_moe_shared_experts_layer = ( - rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name) + self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name) ) name = self._rewrite_spec_layer_name(spec_layer, name) diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index 81e941d284..d5521a1960 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -54,6 +54,10 @@ from vllm.model_executor.layers.fused_moe import ( GateLinear, fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, + resolve_layer_fused_shared_expert, +) from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -329,9 +333,13 @@ class DeepseekV2MoE(nn.Module): self.n_local_physical_experts = self.n_physical_experts // self.ep_size self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled() - self.is_fusion_moe_shared_experts_enabled = ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - ) + + self.is_fused_shared_expert_enabled = False + if config.n_shared_experts is not None: + self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert( + quant_config, prefix + ) + if ( self.is_rocm_aiter_moe_enabled and self.gate.e_score_correction_bias is not None @@ -340,7 +348,7 @@ class DeepseekV2MoE(nn.Module): # Accumulates in fp32; avoids bf16->fp32 cast. self.gate.set_out_dtype(self.gate.weight.dtype) - if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled: + if config.n_shared_experts is None or self.is_fused_shared_expert_enabled: self.shared_experts = None else: intermediate_size = config.moe_intermediate_size * config.n_shared_experts @@ -377,8 +385,9 @@ class DeepseekV2MoE(nn.Module): is_sequence_parallel=self.is_sequence_parallel, reduce_results=reduce_results, n_shared_experts=config.n_shared_experts - if self.is_fusion_moe_shared_experts_enabled + if self.is_fused_shared_expert_enabled else None, + fuse_shared_experts=self.is_fused_shared_expert_enabled, router_logits_dtype=self.gate.out_dtype, ) @@ -1401,6 +1410,12 @@ class DeepseekV2Model(nn.Module): prefix=f"{prefix}.layers", ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + DeepseekV2MoE, + "mlp", + ) + if get_pp_group().is_last_rank: self.norm = RMSNorm(self.hidden_size, eps=config.rms_norm_eps) else: @@ -1511,9 +1526,6 @@ class DeepseekV2Model(nn.Module): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - rocm_aiter_moe_shared_expert_enabled = ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - ) stacked_params_mapping: list[tuple[str, str, int | str]] = [ # (param_name, shard_name, shard_id) ("gate_up_proj", "gate_proj", 0), @@ -1554,7 +1566,7 @@ class DeepseekV2Model(nn.Module): num_experts=self.config.n_routed_experts + ( self.config.n_shared_experts - if rocm_aiter_moe_shared_expert_enabled + if self.is_fused_shared_expert_enabled else 0 ), num_redundant_experts=self.num_redundant_experts, @@ -1582,7 +1594,7 @@ class DeepseekV2Model(nn.Module): continue # this layer has no indexer; drop its checkpoint weights is_fusion_moe_shared_experts_layer = ( - rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name) + self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name) ) if _try_load_fp8_indexer_wk( diff --git a/vllm/model_executor/models/glm4_moe.py b/vllm/model_executor/models/glm4_moe.py index 6272db2a90..2192f74b51 100644 --- a/vllm/model_executor/models/glm4_moe.py +++ b/vllm/model_executor/models/glm4_moe.py @@ -39,12 +39,15 @@ from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_world_size, ) -from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, ) +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, + resolve_layer_fused_shared_expert, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -72,8 +75,6 @@ from .utils import ( skip_spec_layers, ) -logger = init_logger(__name__) - class Glm4MoeMLP(nn.Module): def __init__( @@ -159,14 +160,15 @@ class Glm4MoE(nn.Module): self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts self.n_local_physical_experts = self.n_physical_experts // self.ep_size - # AITER fused shared-expert (FSE) gate; mirrors the deepseek_v2.py - # pattern (see Glm4MoE / MoERunner wiring there). self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled() - self.is_fusion_moe_shared_experts_enabled = ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - ) - if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled: + self.is_fused_shared_expert_enabled = False + if config.n_shared_experts is not None: + self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert( + quant_config, prefix + ) + + if config.n_shared_experts is None or self.is_fused_shared_expert_enabled: self.shared_experts = None else: intermediate_size = config.moe_intermediate_size * config.n_shared_experts @@ -200,10 +202,9 @@ class Glm4MoE(nn.Module): num_redundant_experts=self.n_redundant_experts, router_logits_dtype=torch.float32, n_shared_experts=( - config.n_shared_experts - if self.is_fusion_moe_shared_experts_enabled - else None + config.n_shared_experts if self.is_fused_shared_expert_enabled else None ), + fuse_shared_experts=self.is_fused_shared_expert_enabled, ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -443,6 +444,12 @@ class Glm4MoeModel(nn.Module): prefix=f"{prefix}.layers", ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + Glm4MoE, + "mlp", + ) + if get_pp_group().is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) else: @@ -488,6 +495,7 @@ class Glm4MoeModel(nn.Module): skip_spec_layers(weights, self.config), n_routed_experts=self.config.n_routed_experts, n_shared_experts=self.config.n_shared_experts or 1, + enabled=self.is_fused_shared_expert_enabled, ) loader = AutoWeightsLoader(self) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/glm4_moe_lite.py b/vllm/model_executor/models/glm4_moe_lite.py index 7f63130d88..e51b0e231c 100644 --- a/vllm/model_executor/models/glm4_moe_lite.py +++ b/vllm/model_executor/models/glm4_moe_lite.py @@ -34,7 +34,6 @@ from torch import nn if TYPE_CHECKING: from transformers.models.glm4_moe_lite import Glm4MoeLiteConfig -from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed import ( @@ -44,6 +43,9 @@ from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -254,6 +256,12 @@ class Glm4MoeLiteModel(nn.Module): prefix=f"{prefix}.layers", ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + Glm4MoeLite, + "mlp", + ) + if get_pp_group().is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) else: @@ -320,9 +328,6 @@ class Glm4MoeLiteModel(nn.Module): ) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - rocm_aiter_moe_shared_expert_enabled = ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - ) stacked_params_mapping = [ # (param_name, shard_name, shard_id) ("gate_up_proj", "gate_proj", 0), @@ -345,7 +350,7 @@ class Glm4MoeLiteModel(nn.Module): num_experts=self.config.n_routed_experts + ( self.config.n_shared_experts - if rocm_aiter_moe_shared_expert_enabled + if self.is_fused_shared_expert_enabled else 0 ), ) @@ -361,7 +366,7 @@ class Glm4MoeLiteModel(nn.Module): continue # skip spec decode layers for main model is_fusion_moe_shared_experts_layer = ( - rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name) + self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name) ) for param_name, weight_name, shard_id in stacked_params_mapping: diff --git a/vllm/model_executor/models/glm4_moe_lite_mtp.py b/vllm/model_executor/models/glm4_moe_lite_mtp.py index bbede18c40..6b28c38b58 100644 --- a/vllm/model_executor/models/glm4_moe_lite_mtp.py +++ b/vllm/model_executor/models/glm4_moe_lite_mtp.py @@ -30,12 +30,14 @@ import torch import torch.nn as nn from transformers import PretrainedConfig -from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig from vllm.model_executor.layers.fused_moe import ( MoERunner, fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig @@ -224,6 +226,11 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts): self.moe_mlp_layers.append(layer.mlp) self.moe_layers.append(layer.mlp.experts) self.extract_moe_parameters(example_moe) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.model.layers.values(), + Glm4MoeLite, + "mtp_block.mlp", + ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.model.embed_input_ids(input_ids) @@ -250,9 +257,6 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts): return self.model.compute_logits(hidden_states, spec_step_idx) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - rocm_aiter_moe_shared_expert_enabled = ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - ) stacked_params_mapping = [ ("gate_up_proj", "gate_proj", 0), ("gate_up_proj", "up_proj", 1), @@ -268,7 +272,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts): num_experts=self.config.n_routed_experts + ( self.config.n_shared_experts - if rocm_aiter_moe_shared_expert_enabled + if self.is_fused_shared_expert_enabled else 0 ), ) @@ -282,7 +286,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts): if spec_layer is None: continue is_fusion_moe_shared_experts_layer = ( - rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name) + self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name) ) name = self._rewrite_spec_layer_name(spec_layer, name) for param_name, weight_name, shard_id in stacked_params_mapping: diff --git a/vllm/model_executor/models/glm4_moe_mtp.py b/vllm/model_executor/models/glm4_moe_mtp.py index 6708ba8139..8af40cf60b 100644 --- a/vllm/model_executor/models/glm4_moe_mtp.py +++ b/vllm/model_executor/models/glm4_moe_mtp.py @@ -31,12 +31,14 @@ import torch import torch.nn as nn from transformers import PretrainedConfig -from vllm._aiter_ops import rocm_aiter_ops from vllm.config import CacheConfig, ParallelConfig, VllmConfig from vllm.model_executor.layers.fused_moe import ( MoERunner, fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig @@ -212,6 +214,11 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts): self.moe_mlp_layers.append(layer.mlp) self.moe_layers.append(layer.mlp.experts) self.extract_moe_parameters(example_moe) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.model.layers.values(), + Glm4MoE, + "mtp_block.mlp", + ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.model.embed_input_ids(input_ids) @@ -238,10 +245,6 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts): return self.model.compute_logits(hidden_states, spec_step_idx) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - # FSE weight loading mirrors glm4_moe.py / deepseek_mtp.py. - rocm_aiter_moe_shared_expert_enabled = ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - ) stacked_params_mapping = [ # (param_name, shard_name, shard_id) ("qkv_proj", "q_proj", "q"), @@ -254,7 +257,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) num_experts = self.config.n_routed_experts - if rocm_aiter_moe_shared_expert_enabled and self.config.n_shared_experts: + if self.is_fused_shared_expert_enabled and self.config.n_shared_experts: num_experts += self.config.n_shared_experts expert_params_mapping = fused_moe_make_expert_params_mapping( self, @@ -279,7 +282,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts): name = self._rewrite_spec_layer_name(spec_layer, name) is_fusion_moe_shared_experts_layer = ( - rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name) + self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name) ) for param_name, weight_name, shard_id in stacked_params_mapping: diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 28c6a11899..3fb3b7a782 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -29,13 +29,15 @@ from collections.abc import Iterable import torch from torch import nn -from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed import ( get_pp_group, ) from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) from vllm.model_executor.layers.layernorm import GemmaRMSNorm as Qwen3_5RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( @@ -78,7 +80,6 @@ from .qwen3_next import ( Qwen3NextModel, Qwen3NextSparseMoeBlock, QwenNextMixtureOfExperts, - _is_shared_expert_fse_compatible, ) from .qwen3_vl import ( Qwen3_VisionTransformer, @@ -256,6 +257,11 @@ class Qwen3_5Model(Qwen3NextModel): self.start_layer, self.end_layer, self.layers = make_layers( config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers" ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + Qwen3NextSparseMoeBlock, + "mlp", + ) self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( ["hidden_states", "residual"], config.hidden_size ) @@ -274,8 +280,7 @@ class Qwen3_5Model(Qwen3NextModel): if "moe" in self.config.model_type: weights = maybe_fuse_shared_experts( weights, - enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - and _is_shared_expert_fse_compatible(self.quant_config), + enabled=self.is_fused_shared_expert_enabled, n_routed_experts=self.config.num_experts, n_shared_experts=1, ckpt_prefix="mlp.shared_expert", diff --git a/vllm/model_executor/models/qwen3_5_mtp.py b/vllm/model_executor/models/qwen3_5_mtp.py index 67f06f4f5f..4c064bead7 100644 --- a/vllm/model_executor/models/qwen3_5_mtp.py +++ b/vllm/model_executor/models/qwen3_5_mtp.py @@ -7,11 +7,13 @@ from collections.abc import Iterable import torch from torch import nn -from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.decorators import support_torch_compile -from vllm.config import VllmConfig, get_current_vllm_config +from vllm.config import VllmConfig from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -25,8 +27,8 @@ from vllm.model_executor.models.qwen3_5 import ( Qwen3_5RMSNorm, ) from vllm.model_executor.models.qwen3_next import ( + Qwen3NextSparseMoeBlock, QwenNextMixtureOfExperts, - _is_shared_expert_fse_compatible, ) from vllm.model_executor.models.utils import sequence_parallel_chunk from vllm.sequence import IntermediateTensors @@ -122,6 +124,11 @@ class Qwen3_5MultiTokenPredictor(nn.Module): for idx in range(self.num_mtp_layers) ) vllm_config.quant_config = original_quant + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + Qwen3NextSparseMoeBlock, + "mlp", + ) self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( ["hidden_states", "residual"], config.hidden_size ) @@ -185,10 +192,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module): def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: weights = maybe_fuse_shared_experts( weights, - enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - and _is_shared_expert_fse_compatible( - get_current_vllm_config().quant_config - ), + enabled=self.is_fused_shared_expert_enabled, n_routed_experts=getattr(self.config, "num_experts", 0), n_shared_experts=1, ckpt_prefix="mlp.shared_expert", diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index b241baca6c..9511fcee9e 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -8,7 +8,6 @@ from itertools import islice import torch from torch import nn -from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ModelConfig, VllmConfig from vllm.distributed import ( @@ -18,9 +17,12 @@ from vllm.distributed import ( tensor_model_parallel_all_gather, tensor_model_parallel_reduce_scatter, ) -from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, + resolve_layer_fused_shared_expert, +) from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate from vllm.model_executor.layers.layernorm import ( GemmaRMSNorm as Qwen3NextRMSNorm, @@ -73,8 +75,6 @@ from .utils import ( maybe_prefix, ) -logger = init_logger(__name__) - KVCache = tuple[torch.Tensor, torch.Tensor] @@ -90,26 +90,6 @@ def _should_use_sequence_parallel(vllm_config: VllmConfig) -> bool: ) -def _is_shared_expert_fse_compatible(quant_config) -> bool: - """Check if shared expert can be fused with routed experts. - - FSE requires that shared and routed expert weights use the same - quantization format. Returns False when the shared expert is - excluded from quantization (e.g. float32 shared in an MXFP4 model) - or has a different quant spec than routed experts. - """ - if quant_config is None: - return True - # Quark stores its full config dict in quant_config.quant_config - raw_config = getattr(quant_config, "quant_config", None) - if not isinstance(raw_config, dict): - return True - exclude = raw_config.get("exclude", []) - if not exclude: - return True - return not any("shared_expert." in str(e) for e in exclude) - - class Qwen3NextSparseMoeBlock(nn.Module): def __init__(self, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -157,15 +137,18 @@ class Qwen3NextSparseMoeBlock(nn.Module): prefix=f"{prefix}.shared_expert_gate", ) - _fse_requested = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - _fse_enabled = _fse_requested and _is_shared_expert_fse_compatible(quant_config) - if _fse_requested and not _fse_enabled: - logger.warning( - "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but " - "shared expert has a different quantization spec than routed " - "experts. Falling back to non-fused shared expert path." + self.is_fused_shared_expert_enabled = False + if config.shared_expert_intermediate_size > 0: + self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert( + quant_config, + prefix, + shared_expert_name="shared_expert", ) - if _fse_enabled or config.shared_expert_intermediate_size <= 0: + + if ( + self.is_fused_shared_expert_enabled + or config.shared_expert_intermediate_size <= 0 + ): self.shared_expert = None else: self.shared_expert = Qwen3NextMLP( @@ -193,6 +176,7 @@ class Qwen3NextSparseMoeBlock(nn.Module): num_redundant_experts=self.n_redundant_experts, is_sequence_parallel=self.is_sequence_parallel, n_shared_experts=1 if self.shared_expert is None else None, + fuse_shared_experts=self.is_fused_shared_expert_enabled, shared_expert_gate=self.shared_expert_gate if self.shared_expert is None else None, @@ -599,6 +583,11 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): self.start_layer, self.end_layer, self.layers = make_layers( config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers" ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + Qwen3NextSparseMoeBlock, + "mlp", + ) self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( ["hidden_states", "residual"], config.hidden_size ) @@ -678,6 +667,7 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: weights = maybe_fuse_shared_experts( weights, + enabled=self.is_fused_shared_expert_enabled, n_routed_experts=getattr(self.config, "num_experts", 0), n_shared_experts=1, ckpt_prefix="mlp.shared_expert", diff --git a/vllm/model_executor/models/qwen3_next_mtp.py b/vllm/model_executor/models/qwen3_next_mtp.py index 35f550700a..6b527acd62 100644 --- a/vllm/model_executor/models/qwen3_next_mtp.py +++ b/vllm/model_executor/models/qwen3_next_mtp.py @@ -11,6 +11,9 @@ from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -21,6 +24,7 @@ from vllm.model_executor.models.qwen3_next import ( Qwen3NextDecoderLayer, Qwen3NextModel, Qwen3NextRMSNorm, + Qwen3NextSparseMoeBlock, QwenNextMixtureOfExperts, ) from vllm.model_executor.models.utils import sequence_parallel_chunk @@ -91,6 +95,11 @@ class Qwen3NextMultiTokenPredictor(nn.Module): for idx in range(self.num_mtp_layers) ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + Qwen3NextSparseMoeBlock, + "mlp", + ) self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( ["hidden_states", "residual"], config.hidden_size ) @@ -155,6 +164,7 @@ class Qwen3NextMultiTokenPredictor(nn.Module): def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: weights = maybe_fuse_shared_experts( weights, + enabled=self.is_fused_shared_expert_enabled, n_routed_experts=self.config.num_experts, n_shared_experts=1, ckpt_prefix="mlp.shared_expert", diff --git a/vllm/models/deepseek_v32/amd/mtp.py b/vllm/models/deepseek_v32/amd/mtp.py index 109bd26622..9ea86ee26b 100644 --- a/vllm/models/deepseek_v32/amd/mtp.py +++ b/vllm/models/deepseek_v32/amd/mtp.py @@ -8,12 +8,14 @@ from collections.abc import Callable, Iterable import torch import torch.nn as nn -from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig from vllm.distributed import tensor_model_parallel_all_reduce from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -167,6 +169,11 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") ) self.set_moe_parameters() + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.model.layers.values(), + DeepseekV2MoE, + "mtp_block.mlp", + ) def set_moe_parameters(self): self.num_moe_layers = self.config.num_nextn_predict_layers @@ -231,9 +238,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): return name def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - rocm_aiter_moe_shared_expert_enabled = ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - ) stacked_params_mapping = [ ("gate_up_proj", "gate_proj", 0), ("gate_up_proj", "up_proj", 1), @@ -250,7 +254,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): num_experts=self.config.n_routed_experts + ( self.config.n_shared_experts - if rocm_aiter_moe_shared_expert_enabled + if self.is_fused_shared_expert_enabled else 0 ), ) @@ -266,7 +270,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): if spec_layer is None: continue is_fusion_moe_shared_experts_layer = ( - rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name) + self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name) ) name = self._rewrite_spec_layer_name(spec_layer, name) diff --git a/vllm/models/deepseek_v32/nvidia/mtp.py b/vllm/models/deepseek_v32/nvidia/mtp.py index c2dcbdd64d..2ea01dbdcb 100644 --- a/vllm/models/deepseek_v32/nvidia/mtp.py +++ b/vllm/models/deepseek_v32/nvidia/mtp.py @@ -7,7 +7,6 @@ import torch import torch.nn as nn import vllm.envs as envs -from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.model_executor.layers.fused_embed_norm import ( @@ -344,9 +343,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): return name def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - rocm_aiter_moe_shared_expert_enabled = ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - ) stacked_params_mapping = [ ("gate_up_proj", "gate_proj", 0), ("gate_up_proj", "up_proj", 1), @@ -360,12 +356,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", ckpt_up_proj_name="up_proj", - num_experts=self.config.n_routed_experts - + ( - self.config.n_shared_experts - if rocm_aiter_moe_shared_expert_enabled - else 0 - ), + num_experts=self.config.n_routed_experts, ) pp_missing_layer_names = get_pp_missing_layer_names(self) @@ -378,9 +369,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) if spec_layer is None: continue - is_fusion_moe_shared_experts_layer = ( - rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name) - ) name = self._rewrite_spec_layer_name(spec_layer, name) if _try_load_fp8_indexer_wk( @@ -398,8 +386,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): continue if ("mlp.experts." in name) and name not in params_dict: continue - if is_fusion_moe_shared_experts_layer: - continue name_mapped = name.replace(weight_name, param_name) if ( param_name == "fused_qkv_a_proj" @@ -415,32 +401,10 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): break else: num_chunks = 1 - if is_fusion_moe_shared_experts_layer: - num_chunks = getattr(self.config, "n_shared_experts", 1) or 1 - split_dim = ( - 1 - if ("down_proj.weight" in name and loaded_weight.ndim > 1) - else 0 - ) - total = loaded_weight.shape[split_dim] - assert total % num_chunks == 0 - chunk_size = total // num_chunks for j in range(num_chunks): chunk_name = name weight_to_load = loaded_weight - if is_fusion_moe_shared_experts_layer: - chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size) - if loaded_weight.ndim == 1: - weight_to_load = loaded_weight[chunk_slice] - elif split_dim == 0: - weight_to_load = loaded_weight[chunk_slice, :] - else: - weight_to_load = loaded_weight[:, chunk_slice] - chunk_name = name.replace( - "mlp.shared_experts", - f"mlp.experts.{self.config.n_routed_experts + j}", - ) is_expert_weight = False for mapping in expert_params_mapping: @@ -462,10 +426,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): return_success=True, ) if success: - if not is_fusion_moe_shared_experts_layer: - name = name_mapped - else: - loaded_params.add(name_mapped) + name = name_mapped break else: if is_expert_weight: @@ -485,8 +446,7 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): param, "weight_loader", default_weight_loader ) weight_loader(param, loaded_weight) - if not is_fusion_moe_shared_experts_layer: - loaded_params.add(name) + loaded_params.add(name) loaded_layers: set[int] = set() for param_name in loaded_params: diff --git a/vllm/models/deepseek_v4/amd/model.py b/vllm/models/deepseek_v4/amd/model.py index 5888ae563c..1456408605 100644 --- a/vllm/models/deepseek_v4/amd/model.py +++ b/vllm/models/deepseek_v4/amd/model.py @@ -16,12 +16,16 @@ from vllm.distributed import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, ) +from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, GateLinear, fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -37,6 +41,9 @@ from vllm.model_executor.layers.mhc import ( MHCPreOp, ) from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.utils.config_utils import ( + is_shared_expert_quant_fse_compatible, +) from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, @@ -60,6 +67,8 @@ from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors +logger = init_logger(__name__) + class DeepseekV4MLP(nn.Module): def __init__( @@ -154,48 +163,11 @@ class DeepseekV4MLP(nn.Module): return x -def _shared_experts_are_fp4(config, layer_idx: int | None = None) -> bool: - """Whether the shared experts are MXFP4 and thus fusable. - - ``layer_idx=None`` resolves the model-wide default (global scheme), used by - the main-model weight loader / mapper callers that operate per-model. - """ - quant_cfg = getattr(config, "quantization_config", None) - if quant_cfg is None: - return False - if layer_idx is None: - base = None - elif layer_idx >= config.num_hidden_layers: - base = f"mtp.{layer_idx - config.num_hidden_layers}.ffn.shared_experts" - else: - base = f"layers.{layer_idx}.ffn.shared_experts" - if base and any(e.startswith(base) for e in (quant_cfg.get("exclude") or [])): - return False - entry = ( - (quant_cfg.get("layer_quant_config") or {}).get(f"{base}.w1") if base else None - ) - if entry is None: - entry = quant_cfg.get("global_quant_config") - return ((entry or {}).get("weight") or {}).get("dtype") == "fp4" - - -def _fuse_shared_experts_enabled(config, prefix: str = "") -> bool: - """Whether to fuse the shared expert into the routed MXFP4 grouped GEMM. - - Fusion fuses the shared expert into the routed experts' MXFP4 grouped GEMM, - so it only applies where the shared expert is the same precision as the - routed experts. Some layers may carry a shared expert in a different quantization - than the routed experts; when so, it runs as its own linear and must not be fused. - """ - if not ( - current_platform.is_rocm() - and getattr(config, "n_shared_experts", None) +def _fuse_shared_experts_enabled(config) -> bool: + return bool( + getattr(config, "n_shared_experts", None) and envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS and not get_current_vllm_config().parallel_config.enable_expert_parallel - ): - return False - return _shared_experts_are_fp4( - config, extract_layer_index(prefix) if prefix else None ) @@ -255,9 +227,25 @@ class DeepseekV4MoE(nn.Module): self.n_shared_experts = config.n_shared_experts - self.fuse_shared_experts = _fuse_shared_experts_enabled(config, prefix) + # TODO: Historically, only `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1` + # is checked to enable FSE for DeepSeek-v4, despite AITER not being used. + # This should be cleaned up and use `resolve_layer_fused_shared_expert`. + fse_requested = _fuse_shared_experts_enabled(config) + if fse_requested: + fse_compatible, fse_reason = is_shared_expert_quant_fse_compatible( + quant_config, + f"{prefix}.experts", + f"{prefix}.shared_experts", + ) + if not fse_compatible: + logger.warning( + "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but " + "cannot be enabled: %s.", + fse_reason, + ) + self.is_fused_shared_expert_enabled = fse_requested and fse_compatible - if config.n_shared_experts is None or self.fuse_shared_experts: + if config.n_shared_experts is None or self.is_fused_shared_expert_enabled: self.shared_experts = None else: intermediate_size = config.moe_intermediate_size * config.n_shared_experts @@ -282,8 +270,9 @@ class DeepseekV4MoE(nn.Module): self.experts = FusedMoEFactory( shared_experts=self.shared_experts, n_shared_experts=( - config.n_shared_experts if self.fuse_shared_experts else None + config.n_shared_experts if self.is_fused_shared_expert_enabled else None ), + fuse_shared_experts=self.is_fused_shared_expert_enabled, gate=self.gate, num_experts=config.n_routed_experts, top_k=config.num_experts_per_tok, @@ -581,6 +570,11 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): ), prefix=f"{prefix}.layers", ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + DeepseekV4MoE, + "ffn", + ) if get_pp_group().is_last_rank: self.norm = RMSNorm(config.hidden_size, self.rms_norm_eps) @@ -763,7 +757,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): # diverge from how the module was built if per-layer quantization ever # mixes fused and non-fused layers. fuse_by_layer = { - extract_layer_index(mod_name): mod.fuse_shared_experts + extract_layer_index(mod_name): mod.is_fused_shared_expert_enabled for mod_name, mod in self.named_modules() if isinstance(mod, DeepseekV4MoE) } @@ -874,7 +868,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): # redirected shared-expert weights route through the expert loader. n_shared = getattr(self.config, "n_shared_experts", 0) or 0 num_experts = self.config.n_routed_experts + ( - n_shared if _fuse_shared_experts_enabled(self.config) else 0 + n_shared if self.is_fused_shared_expert_enabled else 0 ) return fused_moe_make_expert_params_mapping( self, @@ -956,15 +950,14 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3): config = vllm_config.model_config.hf_config self.config = config expert_dtype = getattr(config, "expert_dtype", "fp4") - fuse_shared_experts = _fuse_shared_experts_enabled(config) - if expert_dtype != "fp4" or fuse_shared_experts: - self.hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper( - expert_dtype, fuse_shared_experts=fuse_shared_experts - ) - self.model = self.model_cls( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") ) + if expert_dtype != "fp4" or self.model.is_fused_shared_expert_enabled: + self.hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper( + expert_dtype, + fuse_shared_experts=self.model.is_fused_shared_expert_enabled, + ) if get_pp_group().is_last_rank: self.lm_head = ParallelLMHead( config.vocab_size, diff --git a/vllm/models/minimax_m3/amd/model.py b/vllm/models/minimax_m3/amd/model.py index 6aa172f141..798f9a475b 100644 --- a/vllm/models/minimax_m3/amd/model.py +++ b/vllm/models/minimax_m3/amd/model.py @@ -34,6 +34,7 @@ from vllm.config import ( ) from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.forward_context import get_forward_context +from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.attention.attention import set_default_quant_scales from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase @@ -45,6 +46,9 @@ from vllm.model_executor.layers.fused_moe import ( GateLinear, fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, MinimaxM3QKVParallelLinearWithIndexer, @@ -53,6 +57,9 @@ from vllm.model_executor.layers.linear import ( ) from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.utils.config_utils import ( + is_shared_expert_quant_fse_compatible, +) from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -108,23 +115,7 @@ from vllm.v1.kv_cache_interface import ( is_quantized_kv_cache, ) - -def _fuse_shared_experts_enabled(config: PretrainedConfig) -> bool: - """Whether to fuse the shared expert with routed experts. - - ROCm only. Opt-in via ``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS`` (the - router-append fusion runs on both aiter and non-aiter MoE); - it is disabled under expert parallelism (the shared slot is appended to - the routed top-k, which the EP expert-mapping path does not handle). - """ - from vllm.platforms import current_platform - - return bool( - current_platform.is_rocm() - and getattr(config, "n_shared_experts", None) - and envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS - and not get_current_vllm_config().parallel_config.enable_expert_parallel - ) +logger = init_logger(__name__) def _sparse_attention_layer_ids(config: PretrainedConfig) -> set[int]: @@ -303,10 +294,12 @@ class MiniMaxM3MLP(nn.Module): return x -def _aiter_moe_fused_shared_experts_enabled(config: PretrainedConfig) -> bool: +def _aiter_moe_fused_shared_experts_enabled( + is_fused_shared_expert_enabled: bool, +) -> bool: """Whether the fused shared expert routes through aiter's grouped top-k MoE. - A strict sub-case of :func:`_fuse_shared_experts_enabled`: shared-expert + A strict sub-case of `is_fused_shared_expert_enabled`: shared-expert fusion must already be opted in (``VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS``) and allowed (not under expert parallelism). When additionally on gfx950 with an active aiter MoE backend, the shared expert is appended inside aiter's @@ -314,11 +307,13 @@ def _aiter_moe_fused_shared_experts_enabled(config: PretrainedConfig) -> bool: vLLM router's torch concat. Otherwise FSE still runs via the vLLM top-k bias router. """ - if not _fuse_shared_experts_enabled(config): - return False from vllm.platforms.rocm import on_gfx950 - return on_gfx950() and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() + return ( + on_gfx950() + and is_fused_shared_expert_enabled + and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() + ) class MiniMaxM3MoE(nn.Module): @@ -371,11 +366,46 @@ class MiniMaxM3MoE(nn.Module): # MoE call as the last expert slot, so we don't build a separate module. # On gfx950 with aiter MoE the append is fused inside aiter's grouped # top-k kernel; otherwise it goes through the vLLM top-k bias router. - self.fuse_shared_experts = _fuse_shared_experts_enabled(config) - self.use_aiter_moe_fse = _aiter_moe_fused_shared_experts_enabled(config) + # It is disabled under expert parallelism (the shared slot is appended to + # the routed top-k, which the EP expert-mapping path does not handle). + # TODO: Historically, only `VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1` + # is checked to enable FSE for MiniMax-M3, despite AITER not being used. + # This should be cleaned up and use `resolve_layer_fused_shared_expert`. + fse_requested = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS + + self.is_fused_shared_expert_enabled = False + if ( + fse_requested + and bool(getattr(config, "n_shared_experts", None)) + and not get_current_vllm_config().parallel_config.enable_expert_parallel + ): + fse_compatible, fse_reason = is_shared_expert_quant_fse_compatible( + quant_config, + f"{prefix}.experts", + f"{prefix}.shared_experts", + ) + if not fse_compatible: + logger.warning( + "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but " + "cannot be enabled: %s.", + fse_reason, + ) + else: + self.is_fused_shared_expert_enabled = True + + # When additionally on gfx950 with an active aiter MoE backend, the shared + # expert is appended inside aiter's an active aiter MoE backend, the shared + # expert is appended inside aiter's biased grouped top-k kernel + # (``num_fused_shared_experts``) instead of the vLLM router's torch concat. + # Otherwise FSE still runs via the vLLM top-k bias router. + # TODO: `on_gfx950()` check here should not be MiniMax-M3 specific, and + # the check should be done on resolved MOE backend directly. + self.use_aiter_moe_fse = _aiter_moe_fused_shared_experts_enabled( + self.is_fused_shared_expert_enabled + ) self.shared_experts: MiniMaxM3MLP | None = None - if self.n_shared_experts and not self.fuse_shared_experts: + if self.n_shared_experts and not self.is_fused_shared_expert_enabled: self.shared_experts = MiniMaxM3MLP( config=config, intermediate_size=config.intermediate_size * self.n_shared_experts, @@ -412,8 +442,9 @@ class MiniMaxM3MoE(nn.Module): router_logits_dtype=self.gate.out_dtype, shared_experts=self.shared_experts, n_shared_experts=( - self.n_shared_experts if self.fuse_shared_experts else None + self.n_shared_experts if self.is_fused_shared_expert_enabled else None ), + fuse_shared_experts=self.is_fused_shared_expert_enabled, quant_config=quant_config, prefix=f"{prefix}.experts", ) @@ -1101,6 +1132,11 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin): ), prefix=f"{prefix}.layers", ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + MiniMaxM3MoE, + "block_sparse_moe", + ) if get_pp_group().is_last_rank: self.norm = MiniMAXGemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -1155,7 +1191,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin): # expert, include the appended slot (id == num_local_experts). n_shared = getattr(self.config, "n_shared_experts", 0) or 0 num_experts = self.config.num_local_experts + ( - n_shared if _fuse_shared_experts_enabled(self.config) else 0 + n_shared if self.is_fused_shared_expert_enabled else 0 ) return fused_moe_make_expert_params_mapping( self, @@ -1187,8 +1223,6 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin): # (param_name, weight_name, expert_id, shard_id) expert_params_mapping = self.get_expert_mapping() - _fuse_shared = _fuse_shared_experts_enabled(self.config) - params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() for name, loaded_weight in weights: @@ -1206,7 +1240,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin): # down->w2) so it loads via the routed expert loader. Runs before the # stacked/dense mappings so shared_experts.gate_proj/up_proj are not # captured by the dense gate_up_proj mapping. - if _fuse_shared and ".shared_experts." in name: + if self.is_fused_shared_expert_enabled and ".shared_experts." in name: sid = self.config.num_local_experts name = name.replace(".shared_experts.gate_proj.", f".experts.{sid}.w1.") name = name.replace(".shared_experts.up_proj.", f".experts.{sid}.w3.")