[MOE] Standardize and abstract fused shared expert optimization selection (#51695)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-08-18 12:56:56 -04:00
committed by GitHub
parent f9f066d195
commit 88b2bff2c6
20 changed files with 1288 additions and 281 deletions
+22 -10
View File
@@ -46,6 +46,10 @@ from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -148,10 +152,14 @@ class AXK1MoE(nn.Module):
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
self.is_fusion_moe_shared_experts_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled:
self.is_fused_shared_expert_enabled = False
if config.n_shared_experts is not None:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config, prefix
)
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
@@ -189,8 +197,9 @@ class AXK1MoE(nn.Module):
num_redundant_experts=self.n_redundant_experts,
is_sequence_parallel=self.is_sequence_parallel,
n_shared_experts=config.n_shared_experts
if self.is_fusion_moe_shared_experts_enabled
if self.is_fused_shared_expert_enabled
else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -725,6 +734,12 @@ class AXK1Model(nn.Module):
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
AXK1MoE,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
@@ -789,9 +804,6 @@ class AXK1Model(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping: list[tuple[str, str, int | str]] = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "gate_proj", 0),
@@ -822,7 +834,7 @@ class AXK1Model(nn.Module):
num_experts=self.config.n_routed_experts
+ (
(self.config.n_shared_experts or 0)
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
num_redundant_experts=self.num_redundant_experts,
@@ -839,7 +851,7 @@ class AXK1Model(nn.Module):
continue # skip spec decode layers for main model
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
for param_name, weight_name, shard_id in stacked_params_mapping:
+10 -6
View File
@@ -7,13 +7,15 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import tensor_model_parallel_all_gather
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -255,6 +257,11 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
self.moe_mlp_layers.append(layer.mlp)
self.moe_layers.append(layer.mlp.experts)
self.extract_moe_parameters(example_moe)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.model.layers.values(),
DeepseekV2MoE,
"mtp_block.mlp",
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
@@ -285,9 +292,6 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
return self.model.compute_logits(hidden_states, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
@@ -310,7 +314,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
)
@@ -326,7 +330,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
if spec_layer is None:
continue
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
name = self._rewrite_spec_layer_name(spec_layer, name)
+22 -10
View File
@@ -54,6 +54,10 @@ from vllm.model_executor.layers.fused_moe import (
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -329,9 +333,13 @@ class DeepseekV2MoE(nn.Module):
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
self.is_fusion_moe_shared_experts_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
self.is_fused_shared_expert_enabled = False
if config.n_shared_experts is not None:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config, prefix
)
if (
self.is_rocm_aiter_moe_enabled
and self.gate.e_score_correction_bias is not None
@@ -340,7 +348,7 @@ class DeepseekV2MoE(nn.Module):
# Accumulates in fp32; avoids bf16->fp32 cast.
self.gate.set_out_dtype(self.gate.weight.dtype)
if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled:
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
@@ -377,8 +385,9 @@ class DeepseekV2MoE(nn.Module):
is_sequence_parallel=self.is_sequence_parallel,
reduce_results=reduce_results,
n_shared_experts=config.n_shared_experts
if self.is_fusion_moe_shared_experts_enabled
if self.is_fused_shared_expert_enabled
else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
router_logits_dtype=self.gate.out_dtype,
)
@@ -1401,6 +1410,12 @@ class DeepseekV2Model(nn.Module):
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
DeepseekV2MoE,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(self.hidden_size, eps=config.rms_norm_eps)
else:
@@ -1511,9 +1526,6 @@ class DeepseekV2Model(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping: list[tuple[str, str, int | str]] = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "gate_proj", 0),
@@ -1554,7 +1566,7 @@ class DeepseekV2Model(nn.Module):
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
num_redundant_experts=self.num_redundant_experts,
@@ -1582,7 +1594,7 @@ class DeepseekV2Model(nn.Module):
continue # this layer has no indexer; drop its checkpoint weights
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
if _try_load_fp8_indexer_wk(
+20 -12
View File
@@ -39,12 +39,15 @@ from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -72,8 +75,6 @@ from .utils import (
skip_spec_layers,
)
logger = init_logger(__name__)
class Glm4MoeMLP(nn.Module):
def __init__(
@@ -159,14 +160,15 @@ class Glm4MoE(nn.Module):
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
# AITER fused shared-expert (FSE) gate; mirrors the deepseek_v2.py
# pattern (see Glm4MoE / MoERunner wiring there).
self.is_rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
self.is_fusion_moe_shared_experts_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
if config.n_shared_experts is None or self.is_fusion_moe_shared_experts_enabled:
self.is_fused_shared_expert_enabled = False
if config.n_shared_experts is not None:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config, prefix
)
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
@@ -200,10 +202,9 @@ class Glm4MoE(nn.Module):
num_redundant_experts=self.n_redundant_experts,
router_logits_dtype=torch.float32,
n_shared_experts=(
config.n_shared_experts
if self.is_fusion_moe_shared_experts_enabled
else None
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
),
fuse_shared_experts=self.is_fused_shared_expert_enabled,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -443,6 +444,12 @@ class Glm4MoeModel(nn.Module):
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Glm4MoE,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
@@ -488,6 +495,7 @@ class Glm4MoeModel(nn.Module):
skip_spec_layers(weights, self.config),
n_routed_experts=self.config.n_routed_experts,
n_shared_experts=self.config.n_shared_experts or 1,
enabled=self.is_fused_shared_expert_enabled,
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+11 -6
View File
@@ -34,7 +34,6 @@ from torch import nn
if TYPE_CHECKING:
from transformers.models.glm4_moe_lite import Glm4MoeLiteConfig
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import (
@@ -44,6 +43,9 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -254,6 +256,12 @@ class Glm4MoeLiteModel(nn.Module):
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Glm4MoeLite,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
@@ -320,9 +328,6 @@ class Glm4MoeLiteModel(nn.Module):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "gate_proj", 0),
@@ -345,7 +350,7 @@ class Glm4MoeLiteModel(nn.Module):
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
)
@@ -361,7 +366,7 @@ class Glm4MoeLiteModel(nn.Module):
continue # skip spec decode layers for main model
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
for param_name, weight_name, shard_id in stacked_params_mapping:
@@ -30,12 +30,14 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.model_executor.layers.fused_moe import (
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -224,6 +226,11 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
self.moe_mlp_layers.append(layer.mlp)
self.moe_layers.append(layer.mlp.experts)
self.extract_moe_parameters(example_moe)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.model.layers.values(),
Glm4MoeLite,
"mtp_block.mlp",
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
@@ -250,9 +257,6 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
return self.model.compute_logits(hidden_states, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
@@ -268,7 +272,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if rocm_aiter_moe_shared_expert_enabled
if self.is_fused_shared_expert_enabled
else 0
),
)
@@ -282,7 +286,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
if spec_layer is None:
continue
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
name = self._rewrite_spec_layer_name(spec_layer, name)
for param_name, weight_name, shard_id in stacked_params_mapping:
+10 -7
View File
@@ -31,12 +31,14 @@ import torch
import torch.nn as nn
from transformers import PretrainedConfig
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import CacheConfig, ParallelConfig, VllmConfig
from vllm.model_executor.layers.fused_moe import (
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -212,6 +214,11 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
self.moe_mlp_layers.append(layer.mlp)
self.moe_layers.append(layer.mlp.experts)
self.extract_moe_parameters(example_moe)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.model.layers.values(),
Glm4MoE,
"mtp_block.mlp",
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
@@ -238,10 +245,6 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
return self.model.compute_logits(hidden_states, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# FSE weight loading mirrors glm4_moe.py / deepseek_mtp.py.
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
@@ -254,7 +257,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
num_experts = self.config.n_routed_experts
if rocm_aiter_moe_shared_expert_enabled and self.config.n_shared_experts:
if self.is_fused_shared_expert_enabled and self.config.n_shared_experts:
num_experts += self.config.n_shared_experts
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
@@ -279,7 +282,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
name = self._rewrite_spec_layer_name(spec_layer, name)
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
for param_name, weight_name, shard_id in stacked_params_mapping:
+9 -4
View File
@@ -29,13 +29,15 @@ from collections.abc import Iterable
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import (
get_pp_group,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as Qwen3_5RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
@@ -78,7 +80,6 @@ from .qwen3_next import (
Qwen3NextModel,
Qwen3NextSparseMoeBlock,
QwenNextMixtureOfExperts,
_is_shared_expert_fse_compatible,
)
from .qwen3_vl import (
Qwen3_VisionTransformer,
@@ -256,6 +257,11 @@ class Qwen3_5Model(Qwen3NextModel):
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers"
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen3NextSparseMoeBlock,
"mlp",
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
@@ -274,8 +280,7 @@ class Qwen3_5Model(Qwen3NextModel):
if "moe" in self.config.model_type:
weights = maybe_fuse_shared_experts(
weights,
enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
and _is_shared_expert_fse_compatible(self.quant_config),
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=self.config.num_experts,
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
+11 -7
View File
@@ -7,11 +7,13 @@ from collections.abc import Iterable
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -25,8 +27,8 @@ from vllm.model_executor.models.qwen3_5 import (
Qwen3_5RMSNorm,
)
from vllm.model_executor.models.qwen3_next import (
Qwen3NextSparseMoeBlock,
QwenNextMixtureOfExperts,
_is_shared_expert_fse_compatible,
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
@@ -122,6 +124,11 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
for idx in range(self.num_mtp_layers)
)
vllm_config.quant_config = original_quant
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen3NextSparseMoeBlock,
"mlp",
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
@@ -185,10 +192,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = maybe_fuse_shared_experts(
weights,
enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
and _is_shared_expert_fse_compatible(
get_current_vllm_config().quant_config
),
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=getattr(self.config, "num_experts", 0),
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
+22 -32
View File
@@ -8,7 +8,6 @@ from itertools import islice
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.distributed import (
@@ -18,9 +17,12 @@ from vllm.distributed import (
tensor_model_parallel_all_gather,
tensor_model_parallel_reduce_scatter,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate
from vllm.model_executor.layers.layernorm import (
GemmaRMSNorm as Qwen3NextRMSNorm,
@@ -73,8 +75,6 @@ from .utils import (
maybe_prefix,
)
logger = init_logger(__name__)
KVCache = tuple[torch.Tensor, torch.Tensor]
@@ -90,26 +90,6 @@ def _should_use_sequence_parallel(vllm_config: VllmConfig) -> bool:
)
def _is_shared_expert_fse_compatible(quant_config) -> bool:
"""Check if shared expert can be fused with routed experts.
FSE requires that shared and routed expert weights use the same
quantization format. Returns False when the shared expert is
excluded from quantization (e.g. float32 shared in an MXFP4 model)
or has a different quant spec than routed experts.
"""
if quant_config is None:
return True
# Quark stores its full config dict in quant_config.quant_config
raw_config = getattr(quant_config, "quant_config", None)
if not isinstance(raw_config, dict):
return True
exclude = raw_config.get("exclude", [])
if not exclude:
return True
return not any("shared_expert." in str(e) for e in exclude)
class Qwen3NextSparseMoeBlock(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -157,15 +137,18 @@ class Qwen3NextSparseMoeBlock(nn.Module):
prefix=f"{prefix}.shared_expert_gate",
)
_fse_requested = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
_fse_enabled = _fse_requested and _is_shared_expert_fse_compatible(quant_config)
if _fse_requested and not _fse_enabled:
logger.warning(
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
"shared expert has a different quantization spec than routed "
"experts. Falling back to non-fused shared expert path."
self.is_fused_shared_expert_enabled = False
if config.shared_expert_intermediate_size > 0:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config,
prefix,
shared_expert_name="shared_expert",
)
if _fse_enabled or config.shared_expert_intermediate_size <= 0:
if (
self.is_fused_shared_expert_enabled
or config.shared_expert_intermediate_size <= 0
):
self.shared_expert = None
else:
self.shared_expert = Qwen3NextMLP(
@@ -193,6 +176,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
num_redundant_experts=self.n_redundant_experts,
is_sequence_parallel=self.is_sequence_parallel,
n_shared_experts=1 if self.shared_expert is None else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
shared_expert_gate=self.shared_expert_gate
if self.shared_expert is None
else None,
@@ -599,6 +583,11 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers"
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen3NextSparseMoeBlock,
"mlp",
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
@@ -678,6 +667,7 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = maybe_fuse_shared_experts(
weights,
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=getattr(self.config, "num_experts", 0),
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
@@ -11,6 +11,9 @@ from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -21,6 +24,7 @@ from vllm.model_executor.models.qwen3_next import (
Qwen3NextDecoderLayer,
Qwen3NextModel,
Qwen3NextRMSNorm,
Qwen3NextSparseMoeBlock,
QwenNextMixtureOfExperts,
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
@@ -91,6 +95,11 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
for idx in range(self.num_mtp_layers)
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen3NextSparseMoeBlock,
"mlp",
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
@@ -155,6 +164,7 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = maybe_fuse_shared_experts(
weights,
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=self.config.num_experts,
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",