[ROCm][Perf][GLM-5.3-Flash] Enable AITER fused shared experts on gfx950 (#59221)

Signed-off-by: Jin Tao <jin.tao@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com>
This commit is contained in:
Jin Tao
2026-10-07 20:53:40 +08:00
committed by GitHub
co-authored by Cursor Simon Danielsson
parent d9503dc503
commit 2a54f6b625
3 changed files with 213 additions and 6 deletions
+82 -3
View File
@@ -23,6 +23,10 @@ from vllm.model_executor.layers.fused_moe import (
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -258,7 +262,12 @@ class Glm5NextMoE(nn.Module):
)
swiglu_limit = config.swiglu_limit
if config.n_shared_experts is None:
self.is_fused_shared_expert_enabled = False
if config.n_shared_experts is not None:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config, prefix
) and _fused_shared_experts_tuned(parallel_config)
if config.n_shared_experts is None or self.is_fused_shared_expert_enabled:
self.shared_experts = None
else:
intermediate_size = config.moe_intermediate_size * config.n_shared_experts
@@ -294,7 +303,10 @@ class Glm5NextMoE(nn.Module):
enable_eplb=self.enable_eplb,
num_redundant_experts=self.n_redundant_experts,
is_sequence_parallel=self.is_sequence_parallel,
n_shared_experts=None,
n_shared_experts=config.n_shared_experts
if self.is_fused_shared_expert_enabled
else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
router_logits_dtype=self.gate.out_dtype,
swiglu_limit=swiglu_limit,
)
@@ -733,6 +745,9 @@ class Glm5NextModel(nn.Module):
# The active slice is fixed after construction; cache it so forward
# doesn't rebuild the slice (a fresh list) every step.
self._active_layers = self.layers[self.start_layer : self.end_layer]
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers, Glm5NextMoE, "mlp"
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -834,12 +849,15 @@ class Glm5NextModel(nn.Module):
),
0,
)
num_fused_shared = _num_fused_shared_experts(
self.config.n_shared_experts, self.is_fused_shared_expert_enabled
)
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.n_routed_experts,
num_experts=self.config.n_routed_experts + num_fused_shared,
num_redundant_experts=num_redundant_experts,
)
else:
@@ -868,6 +886,8 @@ class Glm5NextModel(nn.Module):
# Models trained using ColossalAI may include these tensors in
# the checkpoint. Skip them.
continue
if self.is_fused_shared_expert_enabled:
name = _fused_shared_expert_name(name, self.config.n_routed_experts)
# Handle FP8 indexer WK: dequantize to BF16 for fusion with
# weights_proj into wk_weights_proj.
@@ -1228,6 +1248,65 @@ def get_spec_layer_idx_from_weight_name(
return None
def _fused_shared_experts_tuned(parallel_config: ParallelConfig) -> bool:
"""AITER has fused-MoE configs tuned for the fused shared-expert shape
(one more expert and one more top-k slot than the routed MoE) only on
gfx950, with every expert on each rank and its weights split by TP4 or
TP8. Data, prefill context and expert parallelism change that split, so
any other GPU or parallel layout would run untuned fallback kernels."""
from vllm.platforms.rocm import on_gfx950
reasons: list[str] = []
if not on_gfx950():
reasons.append("the GPU is not gfx950")
if parallel_config.tensor_parallel_size not in (4, 8):
reasons.append(
f"tensor_parallel_size is {parallel_config.tensor_parallel_size}"
)
if parallel_config.data_parallel_size != 1:
reasons.append(f"data_parallel_size is {parallel_config.data_parallel_size}")
if parallel_config.prefill_context_parallel_size != 1:
reasons.append(
"prefill_context_parallel_size is "
f"{parallel_config.prefill_context_parallel_size}"
)
if parallel_config.enable_expert_parallel:
reasons.append("expert parallelism is enabled")
if not reasons:
return True
logger.warning_once(
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is ignored for GLM-5.3-Flash: "
"%s. AITER has tuned configs for its fused shared-expert MoE only on "
"gfx950 at TP4 and TP8, without data, prefill context or expert "
"parallelism. Running the shared experts as a separate MLP.",
"; ".join(reasons),
)
return False
def _num_fused_shared_experts(n_shared_experts: int | None, enabled: bool) -> int:
"""Expert slots the fused MoE appends for the shared expert; must match the
``num_fused_shared_experts`` that ``FusedMoE`` allocates."""
if not enabled or n_shared_experts is None:
return 0
if n_shared_experts > 1:
raise NotImplementedError(
"Fused shared-expert loading supports only 1 shared expert per "
f"layer, but config.n_shared_experts is {n_shared_experts}. Set "
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 to run the shared "
"experts as a separate MLP."
)
return n_shared_experts
def _fused_shared_expert_name(name: str, n_routed_experts: int) -> str:
"""Point a checkpoint ``mlp.shared_experts.*`` tensor at the fused MoE's
shared-expert slot, which follows the routed experts; other names are
returned unchanged."""
return name.replace("mlp.shared_experts.", f"mlp.experts.{n_routed_experts}.", 1)
def _try_load_fp8_indexer_wk(name, tensor, buf, params_dict, loaded_params):
if "indexer.wk." not in name or "wk_weights" in name:
return False
+14 -1
View File
@@ -10,6 +10,9 @@ from vllm.config import VllmConfig
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -30,6 +33,8 @@ from .model import (
Glm5NextDecoderLayer,
Glm5NextMLAAttention,
Glm5NextMoE,
_fused_shared_expert_name,
_num_fused_shared_experts,
_try_load_fp8_attn_proj,
_try_load_fp8_indexer_wk,
get_spec_layer_idx_from_weight_name,
@@ -217,6 +222,9 @@ class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts):
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.set_moe_parameters()
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.model.layers.values(), Glm5NextMoE, "mtp_block.mlp"
)
def set_moe_parameters(self):
self.num_moe_layers = self.config.num_nextn_predict_layers
@@ -298,12 +306,15 @@ class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts):
("wk_weights_proj", "wk", 0),
("wk_weights_proj", "weights_proj", 1),
]
num_fused_shared = _num_fused_shared_experts(
self.config.n_shared_experts, self.is_fused_shared_expert_enabled
)
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.n_routed_experts,
num_experts=self.config.n_routed_experts + num_fused_shared,
num_redundant_experts=self.num_redundant_experts,
)
@@ -328,6 +339,8 @@ class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts):
if spec_layer is None:
continue
name = self._rewrite_spec_layer_name(spec_layer, name)
if self.is_fused_shared_expert_enabled:
name = _fused_shared_expert_name(name, self.config.n_routed_experts)
if _try_load_fp8_indexer_wk(
name,