[MoE Refactor] Introduce RoutedExperts alias for FusedMoE and don't store SharedExperts in MK (#40735)
Signed-off-by: Bill Nell <bnell@redhat.com> Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Signed-off-by: Robert Shaw <robertgshaw2@gmail.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Robert Shaw <robertgshaw2@gmail.com>
This commit is contained in:
co-authored by
Robert Shaw
Robert Shaw
parent
6ff7405b81
commit
4d591db470
@@ -18,18 +18,15 @@ from vllm.model_executor.kernels.linear import (
|
||||
init_nvfp4_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention, MLAAttention
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.layer import (
|
||||
FusedMoE,
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoeWeightScaleSupported,
|
||||
MoEActivation,
|
||||
RoutedExperts,
|
||||
RoutingMethodType,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
@@ -208,7 +205,7 @@ class ModelOptQuantConfigBase(QuantizationConfig):
|
||||
if getattr(quant_method, "backend", "") == "marlin":
|
||||
quant_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
|
||||
return quant_method
|
||||
elif isinstance(layer, FusedMoE):
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
quant_method = self.FusedMoEMethodCls(
|
||||
quant_config=self, moe_config=layer.moe_config
|
||||
)
|
||||
@@ -779,7 +776,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular,
|
||||
layer: torch.nn.Module,
|
||||
layer: RoutedExperts,
|
||||
) -> mk.FusedMoEExpertsModular:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
@@ -788,7 +785,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
layer: RoutedExperts,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
@@ -867,7 +864,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def _setup_kernel(
|
||||
self,
|
||||
layer: FusedMoE,
|
||||
layer: RoutedExperts,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
@@ -902,10 +899,9 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
shared_experts=layer.shared_experts,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
w13_scale = layer.w13_weight_scale
|
||||
@@ -936,7 +932,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
layer, w13, w2, w13_scale, w2_scale, w13_input_scale, w2_input_scale
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
|
||||
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
|
||||
w1_scale = layer.w13_weight_scale
|
||||
w2_scale = layer.w2_weight_scale
|
||||
a1_scale = layer.w13_input_scale
|
||||
@@ -952,7 +948,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def apply_monolithic(
|
||||
self,
|
||||
layer: FusedMoE,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
@@ -976,10 +972,11 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: FusedMoE,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
assert not self.is_monolithic
|
||||
@@ -994,6 +991,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
|
||||
@@ -1421,7 +1419,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
layer: RoutedExperts,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
@@ -1535,7 +1533,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w2_input_scale", w2_input_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: FusedMoE) -> None:
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
"""
|
||||
Convert NVFP4 MoE weights into kernel format and setup the kernel.
|
||||
"""
|
||||
@@ -1589,12 +1587,11 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
shared_experts=layer.shared_experts,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
|
||||
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
|
||||
return make_nvfp4_moe_quant_config(
|
||||
backend=self.nvfp4_backend,
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
@@ -1611,7 +1608,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
|
||||
def apply_monolithic(
|
||||
self,
|
||||
layer: FusedMoE,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
@@ -1635,10 +1632,11 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: FusedMoE,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
assert not self.is_monolithic
|
||||
@@ -1653,6 +1651,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
|
||||
@@ -1860,15 +1859,15 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
layer: RoutedExperts,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
layer.intermediate_size_per_partition = intermediate_size_per_partition
|
||||
layer.hidden_size = hidden_size
|
||||
assert layer.intermediate_size_per_partition == intermediate_size_per_partition
|
||||
assert layer.hidden_size == hidden_size
|
||||
layer.orig_dtype = params_dtype
|
||||
|
||||
if hidden_size % MXFP8_BLOCK_SIZE != 0:
|
||||
@@ -2051,7 +2050,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
torch.stack(w2_scale_shuffled).contiguous(),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
@@ -2071,7 +2070,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize: mk.FusedMoEPrepareAndFinalizeModular,
|
||||
layer: torch.nn.Module,
|
||||
layer: RoutedExperts,
|
||||
) -> mk.FusedMoEExpertsModular:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
@@ -2079,7 +2078,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
self, layer: RoutedExperts
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
# TRTLLM MXFP8 path is monolithic and does not use modular kernel config.
|
||||
return None
|
||||
@@ -2090,11 +2089,11 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
|
||||
def apply_monolithic(
|
||||
self,
|
||||
layer: FusedMoE,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
from flashinfer.fused_moe.core import (
|
||||
ActivationType,
|
||||
Fp8QuantizationType,
|
||||
@@ -2174,12 +2173,13 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: FusedMoE,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
assert not self.is_monolithic
|
||||
raise NotImplementedError(
|
||||
"Non-monolithic MXFP8 MoE path is not yet implemented."
|
||||
@@ -2295,7 +2295,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
Tries three strategies in order:
|
||||
1. Direct lookup in ``quantized_layers``.
|
||||
2. Packed/fused-layer lookup (unfuse via ``packed_modules_mapping``).
|
||||
3. Prefix-based lookup for FusedMoE (any child key starts with
|
||||
3. Prefix-based lookup for RoutedExperts (any child key starts with
|
||||
``prefix + "."``).
|
||||
|
||||
Returns the upper-cased quant_algo string, or *None* if the prefix
|
||||
@@ -2322,7 +2322,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
f"{algos}. All shards must use the same quantization."
|
||||
)
|
||||
|
||||
# 3. Prefix-based lookup (for FusedMoE / parent modules)
|
||||
# 3. Prefix-based lookup (for RoutedExperts / parent modules)
|
||||
prefix_dot = prefix + "."
|
||||
for key, info in self.quantized_layers.items():
|
||||
if key.startswith(prefix_dot):
|
||||
@@ -2356,7 +2356,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
# Layer not in quantized_layers — leave unquantized
|
||||
return UnquantizedLinearMethod()
|
||||
|
||||
if isinstance(layer, FusedMoE):
|
||||
if isinstance(layer, RoutedExperts):
|
||||
if quant_algo == "FP8":
|
||||
return ModelOptFp8MoEMethod(
|
||||
quant_config=self.fp8_config,
|
||||
|
||||
Reference in New Issue
Block a user