[ROCm][Perf] Add Fused Shared Expert (FSE) support for Qwen3-Next (#39280)
Signed-off-by: nholmber <nholmber@users.noreply.github.com> Signed-off-by: Tres Popp <tres.popp@amd.com> Signed-off-by: Doug Lehr <douglehr@amd.com> Co-authored-by: nholmber <nholmber@users.noreply.github.com> Co-authored-by: Tres <tpopp@users.noreply.github.com> Co-authored-by: Tres Popp <tres.popp@amd.com> Co-authored-by: Doug Lehr <douglehr@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
nholmber
Tres
Tres Popp
Doug Lehr
Cursor
Douglas Lehr
parent
44e6b44a21
commit
2c6b59b807
+69
-2
@@ -261,11 +261,19 @@ def _rocm_aiter_topk_softmax_impl(
|
||||
token_expert_indices: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
renormalize: bool,
|
||||
num_shared_experts: int = 0,
|
||||
shared_expert_scoring_func: str = "",
|
||||
) -> None:
|
||||
from aiter import topk_softmax
|
||||
|
||||
topk_softmax(
|
||||
topk_weights, topk_indices, token_expert_indices, gating_output, renormalize
|
||||
topk_weights,
|
||||
topk_indices,
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
renormalize,
|
||||
num_shared_experts,
|
||||
shared_expert_scoring_func,
|
||||
)
|
||||
|
||||
|
||||
@@ -275,6 +283,8 @@ def _rocm_aiter_topk_softmax_fake(
|
||||
token_expert_indices: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
renormalize: bool,
|
||||
num_shared_experts: int = 0,
|
||||
shared_expert_scoring_func: str = "",
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
@@ -1206,6 +1216,9 @@ class rocm_aiter_ops:
|
||||
_MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
# TODO: Consolidate under _LINEAR_ENABLED
|
||||
_TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
|
||||
# Lazily probed: whether aiter.topk_softmax supports the
|
||||
# num_shared_experts / shared_expert_scoring_func args (7-arg form).
|
||||
_TOPK_SOFTMAX_FUSED_SIGMOID: bool | None = None
|
||||
|
||||
_ALL_REDUCE_MAX_SIZE: int = 8192 * 1024 * 8 * 2
|
||||
_CUSTOM_ALL_REDUCE: AiterCustomAllreduceProto | None = None
|
||||
@@ -1322,6 +1335,52 @@ class rocm_aiter_ops:
|
||||
def is_fusion_moe_shared_experts_enabled(cls) -> bool:
|
||||
return cls.is_fused_moe_enabled() and cls._MOE_SHARED_EXPERTS_ENABLED
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def topk_softmax_supports_fused_sigmoid(cls) -> bool:
|
||||
"""Check if topk_softmax supports fused shared expert activation."""
|
||||
if cls._TOPK_SOFTMAX_FUSED_SIGMOID is None:
|
||||
try:
|
||||
import inspect
|
||||
|
||||
from aiter import topk_softmax
|
||||
|
||||
params = inspect.signature(topk_softmax).parameters
|
||||
if "num_shared_experts" in params:
|
||||
cls._TOPK_SOFTMAX_FUSED_SIGMOID = True
|
||||
else:
|
||||
# @compile_ops wrapper loses the original signature.
|
||||
# Fall back to the torch custom op schema.
|
||||
import torch
|
||||
|
||||
schema = getattr(
|
||||
getattr(torch.ops.aiter, "topk_softmax", None), "default", None
|
||||
)
|
||||
schema_str = str(getattr(schema, "_schema", ""))
|
||||
cls._TOPK_SOFTMAX_FUSED_SIGMOID = "num_shared_experts" in schema_str
|
||||
except (ImportError, ValueError):
|
||||
cls._TOPK_SOFTMAX_FUSED_SIGMOID = False
|
||||
return cls._TOPK_SOFTMAX_FUSED_SIGMOID
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def fuse_sigmoid_in_kernel(cls, aiter_topK_meta_data: object) -> bool:
|
||||
"""Whether fused shared-expert sigmoid in the topk kernel is usable.
|
||||
|
||||
Combines the cached static capability checks (FSE enabled, fused-moe
|
||||
enabled, topk_softmax supports fused sigmoid) with the runtime
|
||||
readiness check (topK meta-data buffer initialized).
|
||||
|
||||
``aiter_topK_meta_data`` is accepted as a parameter rather than
|
||||
imported internally so callers cannot hit initialization-order
|
||||
issues where the module-level global has not been set yet.
|
||||
"""
|
||||
return (
|
||||
cls.is_fusion_moe_shared_experts_enabled()
|
||||
and cls.topk_softmax_supports_fused_sigmoid()
|
||||
and aiter_topK_meta_data is not None
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_mla_enabled(cls) -> bool:
|
||||
@@ -1795,9 +1854,17 @@ class rocm_aiter_ops:
|
||||
token_expert_indices: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
renormalize: bool,
|
||||
num_shared_experts: int = 0,
|
||||
shared_expert_scoring_func: str = "",
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
torch.ops.vllm.rocm_aiter_topk_softmax(
|
||||
topk_weights, topk_indices, token_expert_indices, gating_output, renormalize
|
||||
topk_weights,
|
||||
topk_indices,
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
renormalize,
|
||||
num_shared_experts,
|
||||
shared_expert_scoring_func,
|
||||
)
|
||||
return topk_weights, topk_indices
|
||||
|
||||
|
||||
@@ -285,6 +285,7 @@ class FusedMoE(PluggableLayer):
|
||||
router_logits_dtype: torch.dtype | None = None,
|
||||
gate: torch.nn.Module | None = None,
|
||||
shared_experts: torch.nn.Module | None = None,
|
||||
shared_expert_gate: torch.nn.Module | None = None,
|
||||
routed_input_transform: torch.nn.Module | None = None,
|
||||
routed_output_transform: torch.nn.Module | None = None,
|
||||
apply_routed_scale_to_output: bool = False,
|
||||
@@ -370,6 +371,8 @@ class FusedMoE(PluggableLayer):
|
||||
if n_shared_experts is not None and self.aiter_fmoe_shared_expert_enabled
|
||||
else 0
|
||||
)
|
||||
self.shared_expert_gate = shared_expert_gate
|
||||
|
||||
if (
|
||||
not self.aiter_fmoe_shared_expert_enabled
|
||||
and self.num_fused_shared_experts != 0
|
||||
@@ -608,6 +611,7 @@ class FusedMoE(PluggableLayer):
|
||||
router=self.router,
|
||||
gate=gate,
|
||||
shared_experts=shared_experts,
|
||||
shared_expert_gate=self.shared_expert_gate,
|
||||
quant_method=self.quant_method,
|
||||
enable_dbo=self.vllm_config.parallel_config.enable_dbo,
|
||||
routed_input_transform=routed_input_transform,
|
||||
|
||||
@@ -109,6 +109,55 @@ def init_aiter_topK_meta_data(
|
||||
aiter_topK_meta_data = (total_topk_weights, total_topk_ids)
|
||||
|
||||
|
||||
def inject_shared_expert_weights(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk: int,
|
||||
num_fused_shared_experts: int,
|
||||
shared_expert_weights: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Merge routed topk results with the shared expert buffer and inject
|
||||
dynamic per-token shared expert gate values for AITER fusion.
|
||||
|
||||
For routers that already return the combined buffer (e.g. GroupedTopKRouter
|
||||
via rocm_aiter_grouped_topk), only the dynamic weight injection is needed.
|
||||
For routers that return only routed slots (e.g. FusedTopKRouter), this also
|
||||
copies the routed results into the pre-allocated combined buffer.
|
||||
"""
|
||||
if num_fused_shared_experts == 0:
|
||||
return topk_weights, topk_ids
|
||||
|
||||
assert aiter_topK_meta_data is not None, (
|
||||
"aiter_topK_meta_data is not initialized but "
|
||||
"num_fused_shared_experts > 0. Ensure init_aiter_topK_meta_data "
|
||||
"is called before routing."
|
||||
)
|
||||
|
||||
total_topk_weights, total_topk_ids = aiter_topK_meta_data
|
||||
token = topk_weights.shape[0]
|
||||
|
||||
assert total_topk_weights.shape[0] >= token, (
|
||||
f"AITER topK meta data supports {total_topk_weights.shape[0]} "
|
||||
f"tokens, but got {token} tokens."
|
||||
)
|
||||
|
||||
total_topk_weights_slice = total_topk_weights[:token]
|
||||
total_topk_ids_slice = total_topk_ids[:token]
|
||||
|
||||
if topk_weights.shape[1] == topk:
|
||||
total_topk_weights_slice[:, :topk] = topk_weights
|
||||
total_topk_ids_slice[:, :topk] = topk_ids
|
||||
topk_weights = total_topk_weights_slice
|
||||
topk_ids = total_topk_ids_slice
|
||||
|
||||
if shared_expert_weights is not None:
|
||||
topk_weights[:, topk : topk + num_fused_shared_experts] = shared_expert_weights[
|
||||
:token
|
||||
]
|
||||
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
def rocm_aiter_grouped_topk(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
RoutingMethodType,
|
||||
get_routing_method_type,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import (
|
||||
dispatch_topk_softmax_func,
|
||||
)
|
||||
|
||||
|
||||
class AiterSharedRoutedFusedMoERouter(BaseRouter):
|
||||
"""
|
||||
ROCm AITER router for models with fused shared experts (e.g. Qwen3-MoE).
|
||||
|
||||
When the AITER topk_softmax kernel supports sigmoid fusion, the routing
|
||||
softmax and shared-expert sigmoid are computed in a single kernel launch.
|
||||
Otherwise the shared-expert weights are injected into the pre-allocated
|
||||
AITER buffer via a fallback path.
|
||||
|
||||
Only instantiated when rocm_aiter fused-MoE is active and
|
||||
num_fused_shared_experts > 0.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
top_k: int,
|
||||
global_num_experts: int,
|
||||
eplb_state: EplbLayerState,
|
||||
num_fused_shared_experts: int,
|
||||
scoring_func: str = "softmax",
|
||||
renormalize: bool = True,
|
||||
enable_eplb: bool = False,
|
||||
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
eplb_state=eplb_state,
|
||||
enable_eplb=enable_eplb,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
self.renormalize = renormalize
|
||||
self.scoring_func = scoring_func
|
||||
self.num_fused_shared_experts = num_fused_shared_experts
|
||||
|
||||
@property
|
||||
def routing_method_type(self) -> RoutingMethodType:
|
||||
return get_routing_method_type(
|
||||
scoring_func=self.scoring_func,
|
||||
top_k=self.top_k,
|
||||
renormalize=self.renormalize,
|
||||
num_expert_group=None,
|
||||
has_e_score_bias=False,
|
||||
)
|
||||
|
||||
def _compute_routing(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
indices_type: torch.dtype | None,
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert hidden_states.size(0) == router_logits.size(0), (
|
||||
"Number of tokens mismatch"
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import (
|
||||
aiter_topK_meta_data,
|
||||
)
|
||||
|
||||
M = hidden_states.size(0)
|
||||
topk = self.top_k
|
||||
num_fse = self.num_fused_shared_experts
|
||||
|
||||
token_expert_indices = torch.empty(
|
||||
M, topk, dtype=torch.int32, device=hidden_states.device
|
||||
)
|
||||
|
||||
if rocm_aiter_ops.fuse_sigmoid_in_kernel(aiter_topK_meta_data):
|
||||
total_topk_weights, total_topk_ids = aiter_topK_meta_data # type: ignore[misc]
|
||||
total_topk_weights_slice = total_topk_weights[:M]
|
||||
topk_ids_slice = total_topk_ids[:M, :topk]
|
||||
|
||||
topk_func = dispatch_topk_softmax_func(use_rocm_aiter=True)
|
||||
topk_func(
|
||||
total_topk_weights_slice,
|
||||
topk_ids_slice,
|
||||
token_expert_indices,
|
||||
router_logits,
|
||||
self.renormalize,
|
||||
num_fse,
|
||||
"sigmoid",
|
||||
)
|
||||
return total_topk_weights_slice, total_topk_ids[:M]
|
||||
|
||||
routing_logits = router_logits[:, :-num_fse]
|
||||
shared_logits = router_logits[:, -num_fse:]
|
||||
|
||||
topk_weights = torch.empty(
|
||||
M, topk, dtype=torch.float32, device=hidden_states.device
|
||||
)
|
||||
topk_ids = torch.empty(
|
||||
M,
|
||||
topk,
|
||||
dtype=torch.int32 if indices_type is None else indices_type,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
topk_func = dispatch_topk_softmax_func(
|
||||
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled()
|
||||
)
|
||||
topk_weights, topk_ids = topk_func(
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
token_expert_indices,
|
||||
routing_logits,
|
||||
self.renormalize,
|
||||
)
|
||||
|
||||
if aiter_topK_meta_data is not None:
|
||||
from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import (
|
||||
inject_shared_expert_weights,
|
||||
)
|
||||
|
||||
shared_weights = torch.sigmoid(shared_logits)
|
||||
topk_weights, topk_ids = inject_shared_expert_weights(
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
topk=topk,
|
||||
num_fused_shared_experts=num_fse,
|
||||
shared_expert_weights=shared_weights,
|
||||
)
|
||||
|
||||
return topk_weights, topk_ids
|
||||
@@ -5,8 +5,14 @@ from collections.abc import Callable
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.aiter_shared_routed_fused_moe_router import ( # noqa: E501
|
||||
AiterSharedRoutedFusedMoERouter,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.custom_routing_router import (
|
||||
CustomRoutingRouter,
|
||||
)
|
||||
@@ -67,7 +73,8 @@ def create_fused_moe_router(
|
||||
3. GroupedTopKRouter - if use_grouped_topk is True
|
||||
4. CustomRoutingRouter - if custom_routing_function is not None
|
||||
5. FusedTopKBiasRouter - if e_score_correction_bias is not None
|
||||
6. FusedTopKRouter - default fallback
|
||||
6. AiterSharedRoutedFusedMoERouter - if num_fused_shared_experts > 0
|
||||
7. FusedTopKRouter - default fallback
|
||||
|
||||
Common arguments:
|
||||
top_k: Number of experts to select per token
|
||||
@@ -199,6 +206,22 @@ def create_fused_moe_router(
|
||||
hash_indices_table=hash_indices_table,
|
||||
)
|
||||
|
||||
if (
|
||||
num_fused_shared_experts > 0
|
||||
and scoring_func == "softmax"
|
||||
and rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
):
|
||||
return AiterSharedRoutedFusedMoERouter(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
eplb_state=eplb_state,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
renormalize=renormalize,
|
||||
scoring_func=scoring_func,
|
||||
enable_eplb=enable_eplb,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
|
||||
return FusedTopKRouter(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
|
||||
@@ -223,6 +223,7 @@ class MoERunner(MoERunnerInterface):
|
||||
shared_experts: torch.nn.Module | None,
|
||||
quant_method: FusedMoEMethodBase,
|
||||
enable_dbo: bool,
|
||||
shared_expert_gate: torch.nn.Module | None = None,
|
||||
routed_output_transform: torch.nn.Module | None = None,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
):
|
||||
@@ -233,9 +234,18 @@ class MoERunner(MoERunnerInterface):
|
||||
self.routed_output_transform = routed_output_transform
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.gate = gate
|
||||
self.shared_expert_gate = shared_expert_gate
|
||||
self._quant_method = quant_method
|
||||
self.enable_dbo = enable_dbo
|
||||
|
||||
# When both gates are present and FSE is enabled, fuse their
|
||||
# weight matrices into [num_experts + num_shared, hidden] so one
|
||||
# F.linear produces combined logits. The topk kernel can then
|
||||
# apply routing softmax and shared expert activation (sigmoid)
|
||||
# in a single launch.
|
||||
self._fse_fuse_gate = gate is not None and shared_expert_gate is not None
|
||||
self._combined_gate_weight: torch.Tensor | None = None
|
||||
|
||||
self._shared_experts: SharedExperts | None = None
|
||||
if shared_experts is not None:
|
||||
self._shared_experts = SharedExperts(
|
||||
@@ -278,6 +288,20 @@ class MoERunner(MoERunnerInterface):
|
||||
self._shared_experts._quant_method = quant_method
|
||||
self._quant_method = quant_method
|
||||
|
||||
def _maybe_fuse_gate_weights(self):
|
||||
"""Fuse router and shared expert gate weights on first call.
|
||||
|
||||
Cannot be done at __init__ because gate weights are loaded after
|
||||
module construction (via weight_loader). Called once from
|
||||
_forward_impl before the first forward pass.
|
||||
"""
|
||||
if self._combined_gate_weight is None:
|
||||
assert self.gate is not None and self.shared_expert_gate is not None
|
||||
self._combined_gate_weight = torch.cat(
|
||||
[self.gate.weight, self.shared_expert_gate.weight],
|
||||
dim=0,
|
||||
)
|
||||
|
||||
def is_internal_router(self) -> bool:
|
||||
return self.gate is not None
|
||||
|
||||
@@ -752,7 +776,11 @@ class MoERunner(MoERunnerInterface):
|
||||
# so it can run overlapped with the
|
||||
# NOTE: in future PR, MoE runner will always hold the gate.
|
||||
if self.gate is not None:
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
if self._fse_fuse_gate:
|
||||
self._maybe_fuse_gate_weights()
|
||||
router_logits = F.linear(hidden_states, self._combined_gate_weight)
|
||||
else:
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
|
||||
with self._sequence_parallel_context():
|
||||
# TODO(bnell): parts of the dispatch/combine steps will go away once
|
||||
|
||||
@@ -8,6 +8,7 @@ from itertools import islice
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
@@ -135,7 +136,12 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
prefix=f"{prefix}.shared_expert_gate",
|
||||
)
|
||||
|
||||
if config.shared_expert_intermediate_size > 0:
|
||||
if (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
or config.shared_expert_intermediate_size <= 0
|
||||
):
|
||||
self.shared_expert = None
|
||||
else:
|
||||
self.shared_expert = Qwen3NextMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.shared_expert_intermediate_size,
|
||||
@@ -146,8 +152,6 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
prefix=f"{prefix}.shared_expert",
|
||||
)
|
||||
else:
|
||||
self.shared_expert = None
|
||||
|
||||
self.experts = FusedMoE(
|
||||
shared_experts=self.shared_expert,
|
||||
@@ -162,6 +166,10 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
enable_eplb=self.enable_eplb,
|
||||
num_redundant_experts=self.n_redundant_experts,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
n_shared_experts=1 if self.shared_expert is None else None,
|
||||
shared_expert_gate=self.shared_expert_gate
|
||||
if self.shared_expert is None
|
||||
else None,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -536,12 +544,15 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
num_experts = getattr(self.config, "num_experts", 0)
|
||||
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
|
||||
num_experts += 1
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=getattr(self.config, "num_experts", 0),
|
||||
num_experts=num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
@@ -558,6 +569,10 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
|
||||
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
num_routed = getattr(self.config, "num_experts", 0)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
@@ -571,6 +586,13 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
# FSE: remap shared_expert weights to the fused expert slot
|
||||
if is_fse and "mlp.shared_expert." in name:
|
||||
name = name.replace(
|
||||
"mlp.shared_expert.",
|
||||
f"mlp.experts.{num_routed}.",
|
||||
)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
@@ -7,6 +7,7 @@ from collections.abc import Iterable
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
@@ -147,13 +148,18 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
|
||||
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
num_experts = self.config.num_experts
|
||||
if is_fse:
|
||||
num_experts += 1
|
||||
expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
num_experts=num_experts,
|
||||
)
|
||||
num_routed = self.config.num_experts
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
@@ -161,6 +167,13 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
# FSE: remap shared_expert weights to the fused expert slot
|
||||
if is_fse and "mlp.shared_expert." in name:
|
||||
name = name.replace(
|
||||
"mlp.shared_expert.",
|
||||
f"mlp.experts.{num_routed}.",
|
||||
)
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user