[ROCm][Bugfix][Perf] enable shared expert fusion for Qwen3.5 (#44434)
Signed-off-by: Nico Holmberg <nico.holmberg@amd.com>
This commit is contained in:
@@ -30,6 +30,7 @@ from collections.abc import Callable, Iterable
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
@@ -314,6 +315,15 @@ class Qwen3_5Model(Qwen3NextModel):
|
||||
num_experts = (
|
||||
self.config.num_experts if hasattr(self.config, "num_experts") else 0
|
||||
)
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
from .qwen3_next import _is_shared_expert_fse_compatible
|
||||
|
||||
is_fse = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
and _is_shared_expert_fse_compatible(get_current_vllm_config().quant_config)
|
||||
)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
@@ -327,6 +337,15 @@ class Qwen3_5Model(Qwen3NextModel):
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
# FSE: remap shared_expert weights to fused expert slot
|
||||
if is_fse and "mlp.shared_expert." in name:
|
||||
name = name.replace(
|
||||
"mlp.shared_expert.",
|
||||
f"mlp.experts.{num_experts}.",
|
||||
)
|
||||
is_fused_expert = False
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if "experts.gate_up_proj" in name or "experts.down_proj" in name:
|
||||
is_fused_expert = True
|
||||
|
||||
@@ -88,6 +88,26 @@ logger = init_logger(__name__)
|
||||
KVCache = tuple[torch.Tensor, torch.Tensor]
|
||||
|
||||
|
||||
def _is_shared_expert_fse_compatible(quant_config) -> bool:
|
||||
"""Check if shared expert can be fused with routed experts.
|
||||
|
||||
FSE requires that shared and routed expert weights use the same
|
||||
quantization format. Returns False when the shared expert is
|
||||
excluded from quantization (e.g. float32 shared in an MXFP4 model)
|
||||
or has a different quant spec than routed experts.
|
||||
"""
|
||||
if quant_config is None:
|
||||
return True
|
||||
# Quark stores its full config dict in quant_config.quant_config
|
||||
raw_config = getattr(quant_config, "quant_config", None)
|
||||
if not isinstance(raw_config, dict):
|
||||
return True
|
||||
exclude = raw_config.get("exclude", [])
|
||||
if not exclude:
|
||||
return True
|
||||
return not any("shared_expert." in str(e) for e in exclude)
|
||||
|
||||
|
||||
class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
@@ -142,10 +162,15 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
prefix=f"{prefix}.shared_expert_gate",
|
||||
)
|
||||
|
||||
if (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
or config.shared_expert_intermediate_size <= 0
|
||||
):
|
||||
_fse_requested = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
_fse_enabled = _fse_requested and _is_shared_expert_fse_compatible(quant_config)
|
||||
if _fse_requested and not _fse_enabled:
|
||||
logger.warning(
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
|
||||
"shared expert has a different quantization spec than routed "
|
||||
"experts. Falling back to non-fused shared expert path."
|
||||
)
|
||||
if _fse_enabled or config.shared_expert_intermediate_size <= 0:
|
||||
self.shared_expert = None
|
||||
else:
|
||||
self.shared_expert = Qwen3NextMLP(
|
||||
|
||||
Reference in New Issue
Block a user