[ROCm][Bugfix][Perf] enable shared expert fusion for Qwen3.5 (#44434)

Signed-off-by: Nico Holmberg <nico.holmberg@amd.com>
This commit is contained in:
Nico Holmberg
2026-06-23 23:19:51 +00:00
committed by GitHub
parent 855cd4d787
commit 80e511772f
2 changed files with 48 additions and 4 deletions
+19
View File
@@ -30,6 +30,7 @@ from collections.abc import Callable, Iterable
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import (
@@ -314,6 +315,15 @@ class Qwen3_5Model(Qwen3NextModel):
num_experts = (
self.config.num_experts if hasattr(self.config, "num_experts") else 0
)
from vllm.config import get_current_vllm_config
from .qwen3_next import _is_shared_expert_fse_compatible
is_fse = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
and _is_shared_expert_fse_compatible(get_current_vllm_config().quant_config)
)
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
@@ -327,6 +337,15 @@ class Qwen3_5Model(Qwen3NextModel):
if name is None:
continue
# FSE: remap shared_expert weights to fused expert slot
if is_fse and "mlp.shared_expert." in name:
name = name.replace(
"mlp.shared_expert.",
f"mlp.experts.{num_experts}.",
)
is_fused_expert = False
expert_params_mapping = self.get_expert_mapping()
for param_name, weight_name, shard_id in stacked_params_mapping:
if "experts.gate_up_proj" in name or "experts.down_proj" in name:
is_fused_expert = True
+29 -4
View File
@@ -88,6 +88,26 @@ logger = init_logger(__name__)
KVCache = tuple[torch.Tensor, torch.Tensor]
def _is_shared_expert_fse_compatible(quant_config) -> bool:
"""Check if shared expert can be fused with routed experts.
FSE requires that shared and routed expert weights use the same
quantization format. Returns False when the shared expert is
excluded from quantization (e.g. float32 shared in an MXFP4 model)
or has a different quant spec than routed experts.
"""
if quant_config is None:
return True
# Quark stores its full config dict in quant_config.quant_config
raw_config = getattr(quant_config, "quant_config", None)
if not isinstance(raw_config, dict):
return True
exclude = raw_config.get("exclude", [])
if not exclude:
return True
return not any("shared_expert." in str(e) for e in exclude)
class Qwen3NextSparseMoeBlock(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -142,10 +162,15 @@ class Qwen3NextSparseMoeBlock(nn.Module):
prefix=f"{prefix}.shared_expert_gate",
)
if (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
or config.shared_expert_intermediate_size <= 0
):
_fse_requested = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
_fse_enabled = _fse_requested and _is_shared_expert_fse_compatible(quant_config)
if _fse_requested and not _fse_enabled:
logger.warning(
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
"shared expert has a different quantization spec than routed "
"experts. Falling back to non-fused shared expert path."
)
if _fse_enabled or config.shared_expert_intermediate_size <= 0:
self.shared_expert = None
else:
self.shared_expert = Qwen3NextMLP(