From 9bfd878a48fb92e5e28267f07ad1b7d844ebf160 Mon Sep 17 00:00:00 2001 From: Qiuyang Yue Date: Thu, 25 Jun 2026 06:34:03 -0700 Subject: [PATCH] [MoE] [MoE Refactor] Add moe kernel oracle abc 37753 (#43461) Signed-off-by: Qiuyang Yue Signed-off-by: qyYue1389 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/kernels/moe/test_moe_kernel_oracle.py | 51 +++++++ .../layers/fused_moe/oracle/__init__.py | 10 ++ .../layers/fused_moe/oracle/base.py | 141 ++++++++++++++++++ .../layers/fused_moe/oracle/unquantized.py | 82 +++++++++- .../fused_moe/unquantized_fused_moe_method.py | 2 +- 5 files changed, 280 insertions(+), 6 deletions(-) create mode 100644 tests/kernels/moe/test_moe_kernel_oracle.py create mode 100644 vllm/model_executor/layers/fused_moe/oracle/base.py diff --git a/tests/kernels/moe/test_moe_kernel_oracle.py b/tests/kernels/moe/test_moe_kernel_oracle.py new file mode 100644 index 0000000000..fbdf804a3b --- /dev/null +++ b/tests/kernels/moe/test_moe_kernel_oracle.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the MoEKernelOracle ABC introduced in PR series for #37753. + +This file contains a single canonical demonstration that +`UnquantizedMoEKernelOracle` methods delegate one-to-one to the +existing module-level functions in `oracle/unquantized.py`. Each method +on `UnquantizedMoEKernelOracle` follows the same `return module_fn(args)` +pattern, so verifying delegation for one method (`make_kernel`) gives +high confidence in the rest. +""" + +from unittest.mock import patch + +from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts +from vllm.model_executor.layers.fused_moe.oracle import UnquantizedMoEKernelOracle +from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( + UnquantizedMoeBackend, +) + + +class TestUnquantizedDelegation: + """UnquantizedMoEKernelOracle methods must delegate to the existing + module-level functions; behaviour is bit-identical.""" + + def test_make_kernel_delegates(self) -> None: + quant_config = object() + moe_config = object() + experts_cls = TritonExperts + sentinel_kernel = object() + + with patch( + "vllm.model_executor.layers.fused_moe.oracle.unquantized." + "make_unquantized_moe_kernel", + return_value=sentinel_kernel, + ) as mocked: + out = UnquantizedMoEKernelOracle().make_kernel( + quant_config, + moe_config, + UnquantizedMoeBackend.TRITON, + experts_cls, + ) + + mocked.assert_called_once_with( + quant_config, + moe_config, + UnquantizedMoeBackend.TRITON, + experts_cls, + None, # routing_tables default + ) + assert out is sentinel_kernel diff --git a/vllm/model_executor/layers/fused_moe/oracle/__init__.py b/vllm/model_executor/layers/fused_moe/oracle/__init__.py index 208f01a7cb..f1942819cd 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/__init__.py +++ b/vllm/model_executor/layers/fused_moe/oracle/__init__.py @@ -1,2 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.model_executor.layers.fused_moe.oracle.base import MoEKernelOracle +from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( + UnquantizedMoEKernelOracle, +) + +__all__ = [ + "MoEKernelOracle", + "UnquantizedMoEKernelOracle", +] diff --git a/vllm/model_executor/layers/fused_moe/oracle/base.py b/vllm/model_executor/layers/fused_moe/oracle/base.py new file mode 100644 index 0000000000..6f7b8a9855 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/oracle/base.py @@ -0,0 +1,141 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Abstract base class for MoE kernel oracles. + +Each MoE oracle (unquantized / fp8 / nvfp4 / mxfp4 / mxfp8 / int8 / +int_wna16) is responsible for selecting the right MoE kernel backend for a +given (model, hardware, deployment-config) tuple. The current +implementation expresses this responsibility as module-level functions +that follow an informal convention. + +This module declares the abstract contract; concrete oracles inherit from +`MoEKernelOracle` and provide the platform-specific behaviour. + +This is the first PR in the series suggested by @robertgshaw2-redhat in +PR #37776 (see issue #37753). It intentionally only introduces the ABC; +follow-up PRs migrate each oracle to inherit from it. The single concrete +subclass shipped here (`UnquantizedMoEKernelOracle`) delegates to the +existing module-level functions to keep behaviour bit-identical with +pre-class code. +""" + +from abc import ABC, abstractmethod +from enum import Enum +from typing import TYPE_CHECKING, Generic, TypeVar + +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.config.kernel import MoEBackend +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEQuantConfig, +) + +if TYPE_CHECKING: + from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey + +BackendT = TypeVar("BackendT", bound=Enum) + + +class MoEKernelOracle(ABC, Generic[BackendT]): + """Abstract base for MoE kernel-selection oracles. + + Concrete oracles MUST implement: `backend_enum_cls`, + `get_priority_backends`, `backend_to_kernel_cls`, `map_backend`, + `select_backend`, `make_kernel`. + + Concrete oracles MAY override: `convert_to_kernel_format`, + `make_quant_config`. The base class provides default implementations + that are appropriate for oracles which do not need them + (e.g. `make_quant_config` raises on the unquantized oracle). + """ + + @abstractmethod + def backend_enum_cls(self) -> type[BackendT]: + """Return the concrete `Enum` class enumerating this oracle's + backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`).""" + + @abstractmethod + def get_priority_backends(self, moe_config: FusedMoEConfig) -> list[BackendT]: + """Return platform-appropriate backends in priority order for + this `moe_config`.""" + + @abstractmethod + def backend_to_kernel_cls(self, backend: BackendT) -> type[mk.FusedMoEExperts]: + """Map a backend enum value to its concrete `FusedMoEExperts` + subclass.""" + + @abstractmethod + def map_backend(self, runner_backend: MoEBackend) -> BackendT: + """Map a user-facing `MoEBackend` (from the runner config) to + this oracle's enum.""" + + @abstractmethod + def select_backend( + self, + moe_config: FusedMoEConfig, + weight_key: "QuantKey | None" = None, + activation_key: "QuantKey | None" = None, + ) -> tuple[BackendT, type[mk.FusedMoEExperts] | None]: + """Primary entry point: choose the best supported backend for + the given `moe_config`. + + `weight_key` / `activation_key` carry the quantization scheme of + the weights and activations and are consumed by quantized oracles + (fp8, nvfp4, int8, ...) to disambiguate backends. The unquantized + oracle ignores them. Subclasses with additional selection inputs + (e.g. int_wna16 needs `weight_bits`, fp8 needs + `allow_vllm_cutlass`) widen the signature in their override; a + per-oracle config object is the longer-term target tracked in + the #37753 follow-up PRs. + """ + + @abstractmethod + def make_kernel( + self, + quant_config: FusedMoEQuantConfig, + moe_config: FusedMoEConfig, + backend: BackendT, + experts_cls: type[mk.FusedMoEExperts], + routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, + ) -> mk.FusedMoEKernel: + """Construct the `FusedMoEKernel` (Prepare/Finalize + Experts + combinator) for the chosen backend.""" + + def convert_to_kernel_format( + self, + backend: BackendT, + moe_config: FusedMoEConfig, + w13_weight: torch.Tensor, + w2_weight: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Shuffle weights into the layout expected by `backend`. + + Default implementation returns the inputs unchanged. Oracles + whose backends need weight permutation should override this + (e.g. `UnquantizedMoEKernelOracle` handles AITER and FlashInfer + layouts). + + `moe_config` carries MoE-layer state (e.g. `is_act_and_mul`) + that the conversion needs without coupling the oracle to a + `Module` reference. Quantized oracles whose conversion + additionally needs scales / zero-points / block shapes will + override with a wider signature (and ultimately a per-oracle + config object — tracked in the #37753 follow-up PRs). + """ + return w13_weight, w2_weight + + def make_quant_config(self, *args, **kwargs) -> FusedMoEQuantConfig: + """Build a `FusedMoEQuantConfig` for this oracle. + + Quantized oracles (fp8, nvfp4, mxfp4, ...) override this with + the appropriate signature for their quantization scheme. + Unquantized oracles inherit the default, which raises because + there is no quantization-specific config to build. + """ + raise NotImplementedError( + f"{type(self).__name__} does not implement make_quant_config; " + "this oracle has no quantization-specific config to build." + ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index a7dcd80137..a8ed9c1d7f 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -2,9 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from enum import Enum +from typing import TYPE_CHECKING import torch -from torch.nn import Module import vllm.envs as envs import vllm.model_executor.layers.fused_moe.modular_kernel as mk @@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, ) +from vllm.model_executor.layers.fused_moe.oracle.base import MoEKernelOracle from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( align_moe_weights_for_fi, convert_moe_weights_to_flashinfer_trtllm_block_layout, @@ -25,6 +26,9 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( ) from vllm.platforms import current_platform +if TYPE_CHECKING: + from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey + logger = init_logger(__name__) @@ -256,7 +260,7 @@ def select_unquantized_moe_backend( def convert_to_unquantized_kernel_format( unquantized_backend: UnquantizedMoeBackend, - layer: Module, + moe_config: FusedMoEConfig, w13_weight: torch.Tensor, w2_weight: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: @@ -264,13 +268,13 @@ def convert_to_unquantized_kernel_format( w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight) elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_CUTLASS: - if layer.moe_config.is_act_and_mul: + if moe_config.is_act_and_mul: # Swap halves to arrange as [w3; w1] (kernel expectation) # Non-gated MoE: w13 is a single projection, no need to swap. w13_weight = swap_w13_to_w31(w13_weight) elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM: - is_act_and_mul = layer.moe_config.is_act_and_mul + is_act_and_mul = moe_config.is_act_and_mul if not is_act_and_mul: # Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK # weight layout uses block_k=128). Pad along the intermediate dim when @@ -278,7 +282,7 @@ def convert_to_unquantized_kernel_format( w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi( w13_weight, w2_weight, is_act_and_mul, min_alignment=128 ) - layer.moe_config.intermediate_size_per_partition = padded_intermediate + moe_config.intermediate_size_per_partition = padded_intermediate _cache_permute_indices: dict[torch.Size, torch.Tensor] = {} w13_weight, w2_weight = convert_moe_weights_to_flashinfer_trtllm_block_layout( @@ -333,3 +337,71 @@ def make_unquantized_moe_kernel( ) return kernel + + +# --------------------------------------------------------------------------- +# Class-based view (first PR of the #37753 series; see oracle/base.py). +# Methods delegate to the module-level functions above so behaviour is +# bit-identical with pre-class code. +# --------------------------------------------------------------------------- + + +class UnquantizedMoEKernelOracle(MoEKernelOracle[UnquantizedMoeBackend]): + """Class-based view of the unquantized MoE kernel oracle. + + Each method delegates to its module-level counterpart so that + instantiating and calling this class is bit-identical to calling + the standalone functions. Follow-up PRs may move logic from the + module-level functions into these methods. + """ + + def backend_enum_cls(self) -> type[UnquantizedMoeBackend]: + return UnquantizedMoeBackend + + def get_priority_backends( + self, moe_config: FusedMoEConfig + ) -> list[UnquantizedMoeBackend]: + return _get_priority_backends(moe_config) + + def backend_to_kernel_cls( + self, backend: UnquantizedMoeBackend + ) -> type[mk.FusedMoEExperts]: + return backend_to_kernel_cls(backend) + + def map_backend(self, runner_backend: MoEBackend) -> UnquantizedMoeBackend: + return map_unquantized_backend(runner_backend) + + def select_backend( + self, + moe_config: FusedMoEConfig, + weight_key: "QuantKey | None" = None, + activation_key: "QuantKey | None" = None, + ) -> tuple[UnquantizedMoeBackend, type[mk.FusedMoEExperts] | None]: + assert weight_key is None and activation_key is None, ( + "Weights and activations will never be quantized for " + "UnquantizedMoEKernelOracle" + ) + return select_unquantized_moe_backend(moe_config) + + def convert_to_kernel_format( + self, + backend: UnquantizedMoeBackend, + moe_config: FusedMoEConfig, + w13_weight: torch.Tensor, + w2_weight: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + return convert_to_unquantized_kernel_format( + backend, moe_config, w13_weight, w2_weight + ) + + def make_kernel( + self, + quant_config: FusedMoEQuantConfig, + moe_config: FusedMoEConfig, + backend: UnquantizedMoeBackend, + experts_cls: type[mk.FusedMoEExperts], + routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, + ) -> mk.FusedMoEKernel: + return make_unquantized_moe_kernel( + quant_config, moe_config, backend, experts_cls, routing_tables + ) diff --git a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py index bd4393be5e..7a2c670a8c 100644 --- a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py +++ b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py @@ -161,7 +161,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): # Shuffle weights to runtime format. w13_new, w2_new = convert_to_unquantized_kernel_format( self.unquantized_backend, - layer=layer, + moe_config=layer.moe_config, w13_weight=w13, w2_weight=w2, )