[MOE] Standardize and abstract fused shared expert optimization selection (#51695)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-08-18 12:56:56 -04:00
committed by GitHub
parent f9f066d195
commit 88b2bff2c6
20 changed files with 1288 additions and 281 deletions
+5 -14
View File
@@ -6,7 +6,6 @@ from typing import Any
import torch
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import ParallelConfig, get_current_vllm_config
from vllm.distributed import (
@@ -74,23 +73,13 @@ def determine_expert_counts(
num_experts: int,
num_redundant_experts: int,
n_shared_experts: int | None,
is_act_and_mul: bool,
fuse_shared_experts: bool,
) -> tuple[int, int, int]:
global_num_experts = num_experts + num_redundant_experts
logical_num_experts = num_experts
# Shared-expert fusion: append the shared expert(s) as routed-expert slots
# so they run in the same grouped GEMM. Gated by
# VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: either the native aiter fused-MoE
# path (env + master switch, via is_fusion_moe_shared_experts_enabled) or the
# backend-neutral router-append path (env alone, independent of the master
# switch; e.g. the MM3 triton/flydsl mxfp8 MoE). Gated activations only.
fuse_shared_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
or envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
) and is_act_and_mul
num_fused_shared_experts = (
n_shared_experts if n_shared_experts is not None and fuse_shared_enabled else 0
n_shared_experts if n_shared_experts is not None and fuse_shared_experts else 0
)
return global_num_experts, logical_num_experts, num_fused_shared_experts
@@ -132,6 +121,7 @@ def FusedMoEFactory(
ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"),
is_fused_checkpoint_transposed: bool = False,
n_shared_experts: int | None = None,
fuse_shared_experts: bool = False,
router_logits_dtype: torch.dtype | None = None,
gate: torch.nn.Module | None = None,
shared_experts: torch.nn.Module | None = None,
@@ -198,6 +188,7 @@ def FusedMoEFactory(
block scales use transposed storage.
n_shared_experts: Number of shared experts to fuse into the routed
grouped GEMM (ROCm; requires aiter FSE or the router-append path)
fuse_shared_experts: Whether to enable shared-expert fusion.
router_logits_dtype: Data type for router logits buffers
gate: Pre-configured gate module
shared_experts: Pre-configured shared experts module
@@ -244,7 +235,7 @@ def FusedMoEFactory(
num_experts,
num_redundant_experts,
n_shared_experts,
is_act_and_mul,
fuse_shared_experts,
)
)
@@ -1,15 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from collections.abc import Iterable
from math import prod
from typing import TYPE_CHECKING
import torch
import torch.nn.functional as F
from torch import nn
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.config_utils import (
is_shared_expert_quant_fse_compatible,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
@@ -34,16 +40,95 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
per_tensor_dequantize,
)
from vllm.model_executor.models.utils import PPMissingLayer
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
logger = init_logger(__name__)
def resolve_layer_fused_shared_expert(
quant_config: "QuantizationConfig | None",
prefix: str,
shared_expert_name: str = "shared_experts",
) -> bool:
"""Resolve whether AITER fused shared-expert execution is enabled.
Args:
quant_config: Model quantization configuration.
prefix: MoE module prefix.
shared_expert_name: Shared-expert module name under ``prefix``.
Returns:
Whether AITER fused shared experts are enabled.
Raises:
ValueError: If requested shared-expert fusion is quantization-incompatible.
"""
# NOTE: is_fusion_moe_shared_experts_enabled is decorated with @if_aiter_supported
# that returns None if AITER is not available.
fse_requested = bool(rocm_aiter_ops.is_fusion_moe_shared_experts_enabled())
fse_compatible, fse_reason = (
is_shared_expert_quant_fse_compatible(
quant_config,
f"{prefix}.experts",
f"{prefix}.{shared_expert_name}",
)
if fse_requested
else (True, None)
)
is_fused_shared_expert_enabled = fse_requested and fse_compatible
if fse_requested and not is_fused_shared_expert_enabled:
logger.warning(
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
"cannot be enabled: %s.",
fse_reason,
)
return is_fused_shared_expert_enabled
def is_model_fused_shared_expert_compatible(
layers: nn.ModuleList | Iterable[nn.Module],
moe_cls: type[nn.Module],
moe_name: str,
) -> bool:
"""Resolve one fused-shared-expert state for a model's MoE layers."""
def get_moe_layer(layer: nn.Module) -> nn.Module | None:
for name in moe_name.split("."):
layer = getattr(layer, name, None)
if layer is None:
return None
return layer
moe_layers = (
moe_layer
for layer in layers
if not isinstance(layer, PPMissingLayer)
and (moe_layer := get_moe_layer(layer)) is not None
and isinstance(moe_layer, moe_cls)
)
enabled = [
getattr(layer, "is_fused_shared_expert_enabled", False) for layer in moe_layers
]
enabled_count = sum(enabled)
disabled_count = len(enabled) - enabled_count
if enabled_count > 0 and disabled_count > 0:
raise NotImplementedError(
"Fused shared experts must be enabled for all MoE layers; found "
f"{enabled_count} enabled and {disabled_count} disabled layers. "
"Per-layer fused shared experts is not yet supported. Please open "
"an issue."
)
return enabled_count > 0 and disabled_count == 0
@triton.jit
def _count_expert_num_tokens(
topk_ids_ptr,
@@ -576,63 +576,91 @@ class QuarkConfig(QuantizationConfig):
return True
def _find_matched_config(
self, layer_name: str, module: torch.nn.Module
) -> dict[str, Any]:
def get_layer_quant_config_from_name(
self, layer_name: str
) -> dict[str, Any] | None:
proj_name = layer_name.split(".")[-1]
if proj_name in self.packed_modules_mapping:
shard_proj_names = self.packed_modules_mapping[proj_name]
# Convert fused_name --> [shard_names]
shard_names = [
layer_name.replace(proj_name, shard_proj_name)
for shard_proj_name in shard_proj_names
]
shard_configs = []
for shard_name in shard_names:
if shard_name == layer_name:
config = cast(
dict[str, Any], self.quant_config.get("global_quant_config")
)
for shard_proj_name in shard_proj_names:
shard_name = layer_name.replace(proj_name, shard_proj_name)
if shard_name != layer_name:
config = self.get_layer_quant_config_from_name(shard_name)
else:
config = self._find_matched_config(shard_name, module)
config = None
shard_configs.append(config)
if not all(
deep_compare(q_config, shard_configs[0]) for q_config in shard_configs
matched_configs = [config for config in shard_configs if config is not None]
if matched_configs and not all(
deep_compare(config, matched_configs[0]) for config in matched_configs
):
raise ValueError(
f"Found a different quantization configuration for "
f"{shard_proj_names} in {layer_name}. vLLM "
"requires all to use the same scheme."
)
return shard_configs[0]
if matched_configs:
return matched_configs[0]
return None
else:
layer_quant_config = cast(
dict[str, Any], self.quant_config.get("layer_quant_config")
dict[str, Any], self.quant_config.get("layer_quant_config") or {}
)
def _matches_pattern(layer_name, pattern):
if "*" not in pattern:
return layer_name in pattern
return fnmatch.fnmatch(layer_name, pattern)
for name_pattern, config in layer_quant_config.items():
if _matches_pattern(layer_name, name_pattern):
if "*" not in name_pattern:
matches = layer_name in name_pattern
else:
matches = fnmatch.fnmatch(layer_name, name_pattern)
if matches:
return config
return None
layer_type = cast(str, type(module))
layer_type_quant_config = cast(
dict[str, Any], self.quant_config.get("layer_type_quant_config")
)
if layer_type in layer_type_quant_config:
return layer_type_quant_config[layer_type]
def _find_matched_config(
self, layer_name: str, module: torch.nn.Module
) -> dict[str, Any]:
# Priority order:
# 1. layer_quant_config,
# 2. layer_type_quant_config,
# 3. global_quant_config.
global_quant_config = cast(
dict[str, Any], self.quant_config.get("global_quant_config")
)
return global_quant_config
layer_type = cast(str, type(module))
layer_type_quant_config = cast(
dict[str, Any], self.quant_config.get("layer_type_quant_config")
)
global_quant_config = cast(
dict[str, Any], self.quant_config.get("global_quant_config")
)
fallback_config = layer_type_quant_config.get(layer_type, global_quant_config)
proj_name = layer_name.split(".")[-1]
if proj_name in self.packed_modules_mapping:
shard_proj_names = self.packed_modules_mapping[proj_name]
shard_configs = []
for shard_proj_name in shard_proj_names:
shard_name = layer_name.replace(proj_name, shard_proj_name)
if shard_name == layer_name:
config = fallback_config
else:
config = self.get_layer_quant_config_from_name(shard_name)
if config is None:
config = fallback_config
shard_configs.append(config)
if not all(
deep_compare(config, shard_configs[0]) for config in shard_configs
):
raise ValueError(
f"Found a different quantization configuration for "
f"{shard_proj_names} in {layer_name}. vLLM requires all "
"to use the same scheme."
)
return shard_configs[0]
else:
layer_quant_config = self.get_layer_quant_config_from_name(layer_name)
if layer_quant_config is not None:
return layer_quant_config
return fallback_config
def _get_scheme_from_config(
self, config: dict[str, Any], dynamic_mxfp4_quant: bool = False
@@ -0,0 +1,155 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
import regex as re
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
def is_shared_expert_quant_fse_compatible(
quant_config: "QuantizationConfig | None",
expert_prefix: str,
shared_expert_prefix: str,
projection_names: list[str] | None = None,
) -> tuple[bool, str | None]:
"""Check whether quantization permits fused shared-expert execution.
Args:
quant_config: Model quantization configuration.
expert_prefix: Routed-expert module prefix.
shared_expert_prefix: Shared-expert module prefix.
projection_names: Shared-expert projection names.
Returns:
A compatibility flag and, when incompatible, the reason.
"""
if projection_names is None:
projection_names = ["gate_up_proj", "down_proj"]
if quant_config is None:
return True, None
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
if isinstance(quant_config, DeepseekV4FP8Config):
from vllm.config import get_current_vllm_config
from vllm.model_executor.models.utils import extract_layer_index
if quant_config.expert_dtype != "fp4":
return False, "DeepSeek-V4 routed experts are not MXFP4"
hf_config = get_current_vllm_config().model_config.hf_config
# TODO: This is adapted from former `_shared_experts_are_fp4`, and
# needs to be cleaned up this . There should not be Quark-specific
# logic in DeepseekV4FP8Config.
quantization_config = getattr(hf_config, "quantization_config", None)
if quantization_config is None:
return False, "DeepSeek-V4 has no quantization configuration"
if not quant_config._is_quark_mxfp4_ocp(quantization_config):
return False, "DeepSeek-v4 FSE is only implemented/tested with Quark MXFP4"
layer_idx = extract_layer_index(shared_expert_prefix)
if layer_idx >= hf_config.num_hidden_layers:
shared_expert_prefix = (
f"mtp.{layer_idx - hf_config.num_hidden_layers}.ffn.shared_experts"
)
else:
shared_expert_prefix = f"layers.{layer_idx}.ffn.shared_experts"
if any(
entry.startswith(shared_expert_prefix)
for entry in quantization_config.get("exclude") or []
if isinstance(entry, str)
):
return (
False,
f"DeepSeek-V4 excludes shared experts at {shared_expert_prefix}",
)
shared_expert_weight_name = f"{shared_expert_prefix}.w1"
layer_quant_config = quantization_config.get("layer_quant_config") or {}
layer_config = layer_quant_config.get(shared_expert_weight_name)
if layer_config is None:
layer_config = next(
(
config
for pattern, config in layer_quant_config.items()
if isinstance(pattern, str)
and pattern.startswith("re:")
and re.fullmatch(
pattern.removeprefix("re:"), shared_expert_weight_name
)
),
None,
)
shared_weight_config = (
layer_config or quantization_config.get("global_quant_config") or {}
).get("weight") or {}
if shared_weight_config.get("dtype") == "fp4":
return True, None
return (
False,
f"DeepSeek-V4 shared experts at {shared_expert_prefix} are not MXFP4",
)
if isinstance(quant_config, QuarkConfig):
# TODO: layer_type_quant_config is not taken into account here.
assert "exclude" in quant_config.quant_config
assert "global_quant_config" in quant_config.quant_config
is_compatible = not any(
"shared_expert" in str(entry)
for entry in quant_config.quant_config["exclude"]
)
if not is_compatible:
return False, f"Quark excludes shared experts at {shared_expert_prefix}"
global_quant_config = quant_config.quant_config["global_quant_config"]
def get_projection_quant_configs(layer_name: str) -> list[object]:
module_prefix, _, projection_name = layer_name.rpartition(".")
packed_projection_names = quant_config.packed_modules_mapping.get(
projection_name, [projection_name]
)
return [
quant_config.get_layer_quant_config_from_name(
f"{module_prefix}.{packed_projection_name}"
)
or global_quant_config
for packed_projection_name in packed_projection_names
]
expert_quant_config = (
quant_config.get_layer_quant_config_from_name(expert_prefix)
or global_quant_config
)
shared_expert_quant_configs = [
config
for projection_name in projection_names
for config in get_projection_quant_configs(
f"{shared_expert_prefix}.{projection_name}"
)
]
if all(config == expert_quant_config for config in shared_expert_quant_configs):
return True, None
return (
False,
"Quark uses different quantization configurations for routed and "
f"shared experts at {shared_expert_prefix}",
)
# TODO: Extend FSE support detection to other quantization methods. Typically,
# one would check that the experts and shared_experts use the same
# quantization config. This may be refactored as part of QuantizationConfig later.
return (
False,
"shared-expert FSE quantization compatibility is not implemented for "
f"{type(quant_config).__name__}",
)
+22 -10
View File
@@ -46,6 +46,10 @@ from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -148,10 +152,14 @@ class AXK1MoE(nn.Module):
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
self.is_fusion_moe_shared_experts_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled:
self.is_fused_shared_expert_enabled = False
if config.n_shared_experts is not None:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config, prefix
)
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
@@ -189,8 +197,9 @@ class AXK1MoE(nn.Module):
num_redundant_experts=self.n_redundant_experts,
is_sequence_parallel=self.is_sequence_parallel,
n_shared_experts=config.n_shared_experts
if self.is_fusion_moe_shared_experts_enabled
if self.is_fused_shared_expert_enabled
else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -725,6 +734,12 @@ class AXK1Model(nn.Module):
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
AXK1MoE,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
@@ -789,9 +804,6 @@ class AXK1Model(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping: list[tuple[str, str, int | str]] = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "gate_proj", 0),
@@ -822,7 +834,7 @@ class AXK1Model(nn.Module):
num_experts=self.config.n_routed_experts
+ (
(self.config.n_shared_experts or 0)
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
num_redundant_experts=self.num_redundant_experts,
@@ -839,7 +851,7 @@ class AXK1Model(nn.Module):
continue # skip spec decode layers for main model
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
for param_name, weight_name, shard_id in stacked_params_mapping:
+10 -6
View File
@@ -7,13 +7,15 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import tensor_model_parallel_all_gather
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -255,6 +257,11 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
self.moe_mlp_layers.append(layer.mlp)
self.moe_layers.append(layer.mlp.experts)
self.extract_moe_parameters(example_moe)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.model.layers.values(),
DeepseekV2MoE,
"mtp_block.mlp",
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
@@ -285,9 +292,6 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
return self.model.compute_logits(hidden_states, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
@@ -310,7 +314,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
)
@@ -326,7 +330,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
if spec_layer is None:
continue
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
name = self._rewrite_spec_layer_name(spec_layer, name)
+22 -10
View File
@@ -54,6 +54,10 @@ from vllm.model_executor.layers.fused_moe import (
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -329,9 +333,13 @@ class DeepseekV2MoE(nn.Module):
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
self.is_fusion_moe_shared_experts_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
self.is_fused_shared_expert_enabled = False
if config.n_shared_experts is not None:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config, prefix
)
if (
self.is_rocm_aiter_moe_enabled
and self.gate.e_score_correction_bias is not None
@@ -340,7 +348,7 @@ class DeepseekV2MoE(nn.Module):
# Accumulates in fp32; avoids bf16->fp32 cast.
self.gate.set_out_dtype(self.gate.weight.dtype)
if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled:
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
@@ -377,8 +385,9 @@ class DeepseekV2MoE(nn.Module):
is_sequence_parallel=self.is_sequence_parallel,
reduce_results=reduce_results,
n_shared_experts=config.n_shared_experts
if self.is_fusion_moe_shared_experts_enabled
if self.is_fused_shared_expert_enabled
else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
router_logits_dtype=self.gate.out_dtype,
)
@@ -1401,6 +1410,12 @@ class DeepseekV2Model(nn.Module):
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
DeepseekV2MoE,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(self.hidden_size, eps=config.rms_norm_eps)
else:
@@ -1511,9 +1526,6 @@ class DeepseekV2Model(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping: list[tuple[str, str, int | str]] = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "gate_proj", 0),
@@ -1554,7 +1566,7 @@ class DeepseekV2Model(nn.Module):
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
num_redundant_experts=self.num_redundant_experts,
@@ -1582,7 +1594,7 @@ class DeepseekV2Model(nn.Module):
continue # this layer has no indexer; drop its checkpoint weights
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
if _try_load_fp8_indexer_wk(
+20 -12
View File
@@ -39,12 +39,15 @@ from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -72,8 +75,6 @@ from .utils import (
skip_spec_layers,
)
logger = init_logger(__name__)
class Glm4MoeMLP(nn.Module):
def __init__(
@@ -159,14 +160,15 @@ class Glm4MoE(nn.Module):
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
# AITER fused shared-expert (FSE) gate; mirrors the deepseek_v2.py
# pattern (see Glm4MoE / MoERunner wiring there).
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
self.is_fusion_moe_shared_experts_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled:
self.is_fused_shared_expert_enabled = False
if config.n_shared_experts is not None:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config, prefix
)
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
@@ -200,10 +202,9 @@ class Glm4MoE(nn.Module):
num_redundant_experts=self.n_redundant_experts,
router_logits_dtype=torch.float32,
n_shared_experts=(
config.n_shared_experts
if self.is_fusion_moe_shared_experts_enabled
else None
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
),
fuse_shared_experts=self.is_fused_shared_expert_enabled,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -443,6 +444,12 @@ class Glm4MoeModel(nn.Module):
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Glm4MoE,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
@@ -488,6 +495,7 @@ class Glm4MoeModel(nn.Module):
skip_spec_layers(weights, self.config),
n_routed_experts=self.config.n_routed_experts,
n_shared_experts=self.config.n_shared_experts or 1,
enabled=self.is_fused_shared_expert_enabled,
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+11 -6
View File
@@ -34,7 +34,6 @@ from torch import nn
if TYPE_CHECKING:
from transformers.models.glm4_moe_lite import Glm4MoeLiteConfig
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import (
@@ -44,6 +43,9 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -254,6 +256,12 @@ class Glm4MoeLiteModel(nn.Module):
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Glm4MoeLite,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
@@ -320,9 +328,6 @@ class Glm4MoeLiteModel(nn.Module):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "gate_proj", 0),
@@ -345,7 +350,7 @@ class Glm4MoeLiteModel(nn.Module):
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
)
@@ -361,7 +366,7 @@ class Glm4MoeLiteModel(nn.Module):
continue # skip spec decode layers for main model
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
for param_name, weight_name, shard_id in stacked_params_mapping:
@@ -30,12 +30,14 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.model_executor.layers.fused_moe import (
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -224,6 +226,11 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
self.moe_mlp_layers.append(layer.mlp)
self.moe_layers.append(layer.mlp.experts)
self.extract_moe_parameters(example_moe)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.model.layers.values(),
Glm4MoeLite,
"mtp_block.mlp",
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
@@ -250,9 +257,6 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
return self.model.compute_logits(hidden_states, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
@@ -268,7 +272,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
)
@@ -282,7 +286,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
if spec_layer is None:
continue
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
name = self._rewrite_spec_layer_name(spec_layer, name)
for param_name, weight_name, shard_id in stacked_params_mapping:
+10 -7
View File
@@ -31,12 +31,14 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import CacheConfig, ParallelConfig, VllmConfig
from vllm.model_executor.layers.fused_moe import (
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -212,6 +214,11 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
self.moe_mlp_layers.append(layer.mlp)
self.moe_layers.append(layer.mlp.experts)
self.extract_moe_parameters(example_moe)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.model.layers.values(),
Glm4MoE,
"mtp_block.mlp",
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
@@ -238,10 +245,6 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
return self.model.compute_logits(hidden_states, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# FSE weight loading mirrors glm4_moe.py / deepseek_mtp.py.
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
@@ -254,7 +257,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
num_experts = self.config.n_routed_experts
if rocm_aiter_moe_shared_expert_enabled and self.config.n_shared_experts:
if self.is_fused_shared_expert_enabled and self.config.n_shared_experts:
num_experts += self.config.n_shared_experts
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
@@ -279,7 +282,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
name = self._rewrite_spec_layer_name(spec_layer, name)
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
for param_name, weight_name, shard_id in stacked_params_mapping:
+9 -4
View File
@@ -29,13 +29,15 @@ from collections.abc import Iterable
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import (
get_pp_group,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as Qwen3_5RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
@@ -78,7 +80,6 @@ from .qwen3_next import (
Qwen3NextModel,
Qwen3NextSparseMoeBlock,
QwenNextMixtureOfExperts,
_is_shared_expert_fse_compatible,
)
from .qwen3_vl import (
Qwen3_VisionTransformer,
@@ -256,6 +257,11 @@ class Qwen3_5Model(Qwen3NextModel):
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers"
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen3NextSparseMoeBlock,
"mlp",
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
@@ -274,8 +280,7 @@ class Qwen3_5Model(Qwen3NextModel):
if "moe" in self.config.model_type:
weights = maybe_fuse_shared_experts(
weights,
enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
and _is_shared_expert_fse_compatible(self.quant_config),
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=self.config.num_experts,
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
+11 -7
View File
@@ -7,11 +7,13 @@ from collections.abc import Iterable
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -25,8 +27,8 @@ from vllm.model_executor.models.qwen3_5 import (
Qwen3_5RMSNorm,
)
from vllm.model_executor.models.qwen3_next import (
Qwen3NextSparseMoeBlock,
QwenNextMixtureOfExperts,
_is_shared_expert_fse_compatible,
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
@@ -122,6 +124,11 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
for idx in range(self.num_mtp_layers)
)
vllm_config.quant_config = original_quant
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen3NextSparseMoeBlock,
"mlp",
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
@@ -185,10 +192,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = maybe_fuse_shared_experts(
weights,
enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
and _is_shared_expert_fse_compatible(
get_current_vllm_config().quant_config
),
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=getattr(self.config, "num_experts", 0),
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
+22 -32
View File
@@ -8,7 +8,6 @@ from itertools import islice
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.distributed import (
@@ -18,9 +17,12 @@ from vllm.distributed import (
tensor_model_parallel_all_gather,
tensor_model_parallel_reduce_scatter,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate
from vllm.model_executor.layers.layernorm import (
GemmaRMSNorm as Qwen3NextRMSNorm,
@@ -73,8 +75,6 @@ from .utils import (
maybe_prefix,
)
logger = init_logger(__name__)
KVCache = tuple[torch.Tensor, torch.Tensor]
@@ -90,26 +90,6 @@ def _should_use_sequence_parallel(vllm_config: VllmConfig) -> bool:
)
def _is_shared_expert_fse_compatible(quant_config) -> bool:
"""Check if shared expert can be fused with routed experts.
FSE requires that shared and routed expert weights use the same
quantization format. Returns False when the shared expert is
excluded from quantization (e.g. float32 shared in an MXFP4 model)
or has a different quant spec than routed experts.
"""
if quant_config is None:
return True
# Quark stores its full config dict in quant_config.quant_config
raw_config = getattr(quant_config, "quant_config", None)
if not isinstance(raw_config, dict):
return True
exclude = raw_config.get("exclude", [])
if not exclude:
return True
return not any("shared_expert." in str(e) for e in exclude)
class Qwen3NextSparseMoeBlock(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -157,15 +137,18 @@ class Qwen3NextSparseMoeBlock(nn.Module):
prefix=f"{prefix}.shared_expert_gate",
)
_fse_requested = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
_fse_enabled = _fse_requested and _is_shared_expert_fse_compatible(quant_config)
if _fse_requested and not _fse_enabled:
logger.warning(
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
"shared expert has a different quantization spec than routed "
"experts. Falling back to non-fused shared expert path."
self.is_fused_shared_expert_enabled = False
if config.shared_expert_intermediate_size > 0:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config,
prefix,
shared_expert_name="shared_expert",
)
if _fse_enabled or config.shared_expert_intermediate_size <= 0:
if (
self.is_fused_shared_expert_enabled
or config.shared_expert_intermediate_size <= 0
):
self.shared_expert = None
else:
self.shared_expert = Qwen3NextMLP(
@@ -193,6 +176,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
num_redundant_experts=self.n_redundant_experts,
is_sequence_parallel=self.is_sequence_parallel,
n_shared_experts=1 if self.shared_expert is None else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
shared_expert_gate=self.shared_expert_gate
if self.shared_expert is None
else None,
@@ -599,6 +583,11 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers"
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen3NextSparseMoeBlock,
"mlp",
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
@@ -678,6 +667,7 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = maybe_fuse_shared_experts(
weights,
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=getattr(self.config, "num_experts", 0),
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
@@ -11,6 +11,9 @@ from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -21,6 +24,7 @@ from vllm.model_executor.models.qwen3_next import (
Qwen3NextDecoderLayer,
Qwen3NextModel,
Qwen3NextRMSNorm,
Qwen3NextSparseMoeBlock,
QwenNextMixtureOfExperts,
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
@@ -91,6 +95,11 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
for idx in range(self.num_mtp_layers)
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen3NextSparseMoeBlock,
"mlp",
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
@@ -155,6 +164,7 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = maybe_fuse_shared_experts(
weights,
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=self.config.num_experts,
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",