[MoE Refactor] Introduce RoutedExperts alias for FusedMoE and don't store SharedExperts in MK (#40735)
Signed-off-by: Bill Nell <bnell@redhat.com> Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Signed-off-by: Robert Shaw <robertgshaw2@gmail.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Robert Shaw <robertgshaw2@gmail.com>
This commit is contained in:
co-authored by
Robert Shaw
Robert Shaw
parent
6ff7405b81
commit
4d591db470
@@ -4,7 +4,6 @@
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from torch.nn import Module
|
||||
from torch.utils._python_dispatch import TorchDispatchMode
|
||||
|
||||
import vllm.envs as envs
|
||||
@@ -22,14 +21,15 @@ from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
FusedMoEMethodBase,
|
||||
FusedMoeWeightScaleSupported,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.layer import UnquantizedFusedMoEMethod
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
@@ -191,7 +191,7 @@ class Fp8Config(QuantizationConfig):
|
||||
offline_method = Fp8LinearMethod(self)
|
||||
offline_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
|
||||
return offline_method
|
||||
elif isinstance(layer, FusedMoE):
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
if is_layer_skipped(
|
||||
prefix=prefix,
|
||||
ignored_layers=self.ignored_layers,
|
||||
@@ -315,7 +315,7 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
layer: RoutedExperts,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
@@ -389,7 +389,7 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
|
||||
self.use_marlin = isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
if self.use_marlin:
|
||||
# Only Marlin kernels support `marlin_input_dtype`; guard to avoid
|
||||
# AttributeError if backend selection changes.
|
||||
@@ -534,7 +534,7 @@ class Fp8OnlineLinearMethod(Fp8LinearMethod):
|
||||
)
|
||||
self.use_marlin = isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
@@ -575,7 +575,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
quant_config: The quantization config.
|
||||
"""
|
||||
|
||||
def __init__(self, quant_config: Fp8Config, layer: torch.nn.Module):
|
||||
def __init__(self, quant_config: Fp8Config, layer: RoutedExperts):
|
||||
super().__init__(layer.moe_config)
|
||||
self.quant_config = quant_config
|
||||
self.weight_block_size = self.quant_config.weight_block_size
|
||||
@@ -606,7 +606,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: Module,
|
||||
layer: RoutedExperts,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
@@ -746,7 +746,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def _setup_kernel(
|
||||
self,
|
||||
layer: FusedMoE,
|
||||
layer: RoutedExperts,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
@@ -787,10 +787,9 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
shared_experts=layer.shared_experts,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
# Allow for accessing weights and scales in standard way.
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
@@ -844,7 +843,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
"logic. This function should not be called."
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
|
||||
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
|
||||
w1_scale = getattr(layer, f"w13_{self.weight_scale_name}")
|
||||
w2_scale = getattr(layer, f"w2_{self.weight_scale_name}")
|
||||
a1_scale = layer.w13_input_scale
|
||||
@@ -877,7 +876,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def apply_monolithic(
|
||||
self,
|
||||
layer: FusedMoE,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
@@ -901,10 +900,11 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: FusedMoE,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
assert not self.is_monolithic
|
||||
@@ -919,6 +919,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
|
||||
@@ -937,7 +938,7 @@ class Fp8OnlineMoEMethod(Fp8MoEMethod):
|
||||
|
||||
uses_meta_device: bool = True
|
||||
|
||||
def __init__(self, quant_config: Fp8Config, layer: torch.nn.Module):
|
||||
def __init__(self, quant_config: Fp8Config, layer: RoutedExperts):
|
||||
super().__init__(quant_config, layer)
|
||||
assert not quant_config.is_checkpoint_fp8_serialized
|
||||
assert quant_config.activation_scheme == "dynamic"
|
||||
@@ -945,7 +946,7 @@ class Fp8OnlineMoEMethod(Fp8MoEMethod):
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: Module,
|
||||
layer: RoutedExperts,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
@@ -1011,7 +1012,7 @@ class Fp8OnlineMoEMethod(Fp8MoEMethod):
|
||||
|
||||
initialize_online_processing(layer)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
# TODO(@ksayers): inplace fp8 quant kernel, initialize scales with ones
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user