[MoE Refactor] Introduce RoutedExperts alias for FusedMoE and don't store SharedExperts in MK (#40735)

Signed-off-by: Bill Nell <bnell@redhat.com>
Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Signed-off-by: Robert Shaw <robertgshaw2@gmail.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: Robert Shaw <robertgshaw2@gmail.com>
This commit is contained in:
bnellnm
2026-05-12 13:37:44 -04:00
committed by GitHub
co-authored by Robert Shaw Robert Shaw
parent 6ff7405b81
commit 4d591db470
53 changed files with 369 additions and 332 deletions
@@ -18,18 +18,15 @@ from vllm.model_executor.kernels.linear import (
init_nvfp4_linear_kernel,
)
from vllm.model_executor.layers.attention import Attention, MLAAttention
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.layer import (
FusedMoE,
FusedMoEQuantConfig,
FusedMoeWeightScaleSupported,
MoEActivation,
RoutedExperts,
RoutingMethodType,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
@@ -208,7 +205,7 @@ class ModelOptQuantConfigBase(QuantizationConfig):
if getattr(quant_method, "backend", "") == "marlin":
quant_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
return quant_method
elif isinstance(layer, FusedMoE):
elif isinstance(layer, RoutedExperts):
quant_method = self.FusedMoEMethodCls(
quant_config=self, moe_config=layer.moe_config
)
@@ -779,7 +776,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
def select_gemm_impl(
self,
prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular,
layer: torch.nn.Module,
layer: RoutedExperts,
) -> mk.FusedMoEExpertsModular:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
@@ -788,7 +785,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
def create_weights(
self,
layer: torch.nn.Module,
layer: RoutedExperts,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
@@ -867,7 +864,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
def _setup_kernel(
self,
layer: FusedMoE,
layer: RoutedExperts,
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
@@ -902,10 +899,9 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
shared_experts=layer.shared_experts,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
w13 = layer.w13_weight
w2 = layer.w2_weight
w13_scale = layer.w13_weight_scale
@@ -936,7 +932,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
layer, w13, w2, w13_scale, w2_scale, w13_input_scale, w2_input_scale
)
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
w1_scale = layer.w13_weight_scale
w2_scale = layer.w2_weight_scale
a1_scale = layer.w13_input_scale
@@ -952,7 +948,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
def apply_monolithic(
self,
layer: FusedMoE,
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
@@ -976,10 +972,11 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
def apply(
self,
layer: FusedMoE,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
assert not self.is_monolithic
@@ -994,6 +991,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -1421,7 +1419,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
def create_weights(
self,
layer: torch.nn.Module,
layer: RoutedExperts,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
@@ -1535,7 +1533,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
)
layer.register_parameter("w2_input_scale", w2_input_scale)
def process_weights_after_loading(self, layer: FusedMoE) -> None:
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
"""
Convert NVFP4 MoE weights into kernel format and setup the kernel.
"""
@@ -1589,12 +1587,11 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
shared_experts=layer.shared_experts,
routing_tables=layer._expert_routing_tables(),
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
return make_nvfp4_moe_quant_config(
backend=self.nvfp4_backend,
w13_scale=layer.w13_weight_scale,
@@ -1611,7 +1608,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
def apply_monolithic(
self,
layer: FusedMoE,
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
@@ -1635,10 +1632,11 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
def apply(
self,
layer: FusedMoE,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
assert not self.is_monolithic
@@ -1653,6 +1651,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -1860,15 +1859,15 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
def create_weights(
self,
layer: torch.nn.Module,
layer: RoutedExperts,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
layer.intermediate_size_per_partition = intermediate_size_per_partition
layer.hidden_size = hidden_size
assert layer.intermediate_size_per_partition == intermediate_size_per_partition
assert layer.hidden_size == hidden_size
layer.orig_dtype = params_dtype
if hidden_size % MXFP8_BLOCK_SIZE != 0:
@@ -2051,7 +2050,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
torch.stack(w2_scale_shuffled).contiguous(),
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
@@ -2071,7 +2070,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
def select_gemm_impl(
self,
prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular,
layer: torch.nn.Module,
layer: RoutedExperts,
) -> mk.FusedMoEExpertsModular:
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
@@ -2079,7 +2078,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
)
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
self, layer: RoutedExperts
) -> FusedMoEQuantConfig | None:
# TRTLLM MXFP8 path is monolithic and does not use modular kernel config.
return None
@@ -2090,11 +2089,11 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
def apply_monolithic(
self,
layer: FusedMoE,
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
from flashinfer.fused_moe.core import (
ActivationType,
Fp8QuantizationType,
@@ -2174,12 +2173,13 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
def apply(
self,
layer: FusedMoE,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
assert not self.is_monolithic
raise NotImplementedError(
"Non-monolithic MXFP8 MoE path is not yet implemented."
@@ -2295,7 +2295,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
Tries three strategies in order:
1. Direct lookup in ``quantized_layers``.
2. Packed/fused-layer lookup (unfuse via ``packed_modules_mapping``).
3. Prefix-based lookup for FusedMoE (any child key starts with
3. Prefix-based lookup for RoutedExperts (any child key starts with
``prefix + "."``).
Returns the upper-cased quant_algo string, or *None* if the prefix
@@ -2322,7 +2322,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
f"{algos}. All shards must use the same quantization."
)
# 3. Prefix-based lookup (for FusedMoE / parent modules)
# 3. Prefix-based lookup (for RoutedExperts / parent modules)
prefix_dot = prefix + "."
for key, info in self.quantized_layers.items():
if key.startswith(prefix_dot):
@@ -2356,7 +2356,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
# Layer not in quantized_layers — leave unquantized
return UnquantizedLinearMethod()
if isinstance(layer, FusedMoE):
if isinstance(layer, RoutedExperts):
if quant_algo == "FP8":
return ModelOptFp8MoEMethod(
quant_config=self.fp8_config,