[MoE] [MoE Refactor] Add moe kernel oracle abc 37753 (#43461)

Signed-off-by: Qiuyang Yue <yueqiuyang1389@gmail.com>
Signed-off-by: qyYue1389 <yueqiuyang1389@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Qiuyang Yue
2026-06-25 09:34:03 -04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 2365b7a8e7
commit 9bfd878a48
5 changed files with 280 additions and 6 deletions
@@ -0,0 +1,51 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the MoEKernelOracle ABC introduced in PR series for #37753.
This file contains a single canonical demonstration that
`UnquantizedMoEKernelOracle` methods delegate one-to-one to the
existing module-level functions in `oracle/unquantized.py`. Each method
on `UnquantizedMoEKernelOracle` follows the same `return module_fn(args)`
pattern, so verifying delegation for one method (`make_kernel`) gives
high confidence in the rest.
"""
from unittest.mock import patch
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
from vllm.model_executor.layers.fused_moe.oracle import UnquantizedMoEKernelOracle
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
)
class TestUnquantizedDelegation:
"""UnquantizedMoEKernelOracle methods must delegate to the existing
module-level functions; behaviour is bit-identical."""
def test_make_kernel_delegates(self) -> None:
quant_config = object()
moe_config = object()
experts_cls = TritonExperts
sentinel_kernel = object()
with patch(
"vllm.model_executor.layers.fused_moe.oracle.unquantized."
"make_unquantized_moe_kernel",
return_value=sentinel_kernel,
) as mocked:
out = UnquantizedMoEKernelOracle().make_kernel(
quant_config,
moe_config,
UnquantizedMoeBackend.TRITON,
experts_cls,
)
mocked.assert_called_once_with(
quant_config,
moe_config,
UnquantizedMoeBackend.TRITON,
experts_cls,
None, # routing_tables default
)
assert out is sentinel_kernel
@@ -1,2 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.layers.fused_moe.oracle.base import MoEKernelOracle
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoEKernelOracle,
)
__all__ = [
"MoEKernelOracle",
"UnquantizedMoEKernelOracle",
]
@@ -0,0 +1,141 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Abstract base class for MoE kernel oracles.
Each MoE oracle (unquantized / fp8 / nvfp4 / mxfp4 / mxfp8 / int8 /
int_wna16) is responsible for selecting the right MoE kernel backend for a
given (model, hardware, deployment-config) tuple. The current
implementation expresses this responsibility as module-level functions
that follow an informal convention.
This module declares the abstract contract; concrete oracles inherit from
`MoEKernelOracle` and provide the platform-specific behaviour.
This is the first PR in the series suggested by @robertgshaw2-redhat in
PR #37776 (see issue #37753). It intentionally only introduces the ABC;
follow-up PRs migrate each oracle to inherit from it. The single concrete
subclass shipped here (`UnquantizedMoEKernelOracle`) delegates to the
existing module-level functions to keep behaviour bit-identical with
pre-class code.
"""
from abc import ABC, abstractmethod
from enum import Enum
from typing import TYPE_CHECKING, Generic, TypeVar
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config.kernel import MoEBackend
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
BackendT = TypeVar("BackendT", bound=Enum)
class MoEKernelOracle(ABC, Generic[BackendT]):
"""Abstract base for MoE kernel-selection oracles.
Concrete oracles MUST implement: `backend_enum_cls`,
`get_priority_backends`, `backend_to_kernel_cls`, `map_backend`,
`select_backend`, `make_kernel`.
Concrete oracles MAY override: `convert_to_kernel_format`,
`make_quant_config`. The base class provides default implementations
that are appropriate for oracles which do not need them
(e.g. `make_quant_config` raises on the unquantized oracle).
"""
@abstractmethod
def backend_enum_cls(self) -> type[BackendT]:
"""Return the concrete `Enum` class enumerating this oracle's
backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`)."""
@abstractmethod
def get_priority_backends(self, moe_config: FusedMoEConfig) -> list[BackendT]:
"""Return platform-appropriate backends in priority order for
this `moe_config`."""
@abstractmethod
def backend_to_kernel_cls(self, backend: BackendT) -> type[mk.FusedMoEExperts]:
"""Map a backend enum value to its concrete `FusedMoEExperts`
subclass."""
@abstractmethod
def map_backend(self, runner_backend: MoEBackend) -> BackendT:
"""Map a user-facing `MoEBackend` (from the runner config) to
this oracle's enum."""
@abstractmethod
def select_backend(
self,
moe_config: FusedMoEConfig,
weight_key: "QuantKey | None" = None,
activation_key: "QuantKey | None" = None,
) -> tuple[BackendT, type[mk.FusedMoEExperts] | None]:
"""Primary entry point: choose the best supported backend for
the given `moe_config`.
`weight_key` / `activation_key` carry the quantization scheme of
the weights and activations and are consumed by quantized oracles
(fp8, nvfp4, int8, ...) to disambiguate backends. The unquantized
oracle ignores them. Subclasses with additional selection inputs
(e.g. int_wna16 needs `weight_bits`, fp8 needs
`allow_vllm_cutlass`) widen the signature in their override; a
per-oracle config object is the longer-term target tracked in
the #37753 follow-up PRs.
"""
@abstractmethod
def make_kernel(
self,
quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
backend: BackendT,
experts_cls: type[mk.FusedMoEExperts],
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEKernel:
"""Construct the `FusedMoEKernel` (Prepare/Finalize + Experts
combinator) for the chosen backend."""
def convert_to_kernel_format(
self,
backend: BackendT,
moe_config: FusedMoEConfig,
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Shuffle weights into the layout expected by `backend`.
Default implementation returns the inputs unchanged. Oracles
whose backends need weight permutation should override this
(e.g. `UnquantizedMoEKernelOracle` handles AITER and FlashInfer
layouts).
`moe_config` carries MoE-layer state (e.g. `is_act_and_mul`)
that the conversion needs without coupling the oracle to a
`Module` reference. Quantized oracles whose conversion
additionally needs scales / zero-points / block shapes will
override with a wider signature (and ultimately a per-oracle
config object — tracked in the #37753 follow-up PRs).
"""
return w13_weight, w2_weight
def make_quant_config(self, *args, **kwargs) -> FusedMoEQuantConfig:
"""Build a `FusedMoEQuantConfig` for this oracle.
Quantized oracles (fp8, nvfp4, mxfp4, ...) override this with
the appropriate signature for their quantization scheme.
Unquantized oracles inherit the default, which raises because
there is no quantization-specific config to build.
"""
raise NotImplementedError(
f"{type(self).__name__} does not implement make_quant_config; "
"this oracle has no quantization-specific config to build."
)
@@ -2,9 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
from typing import TYPE_CHECKING
import torch
from torch.nn import Module
import vllm.envs as envs
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.oracle.base import MoEKernelOracle
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
align_moe_weights_for_fi,
convert_moe_weights_to_flashinfer_trtllm_block_layout,
@@ -25,6 +26,9 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
)
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
logger = init_logger(__name__)
@@ -256,7 +260,7 @@ def select_unquantized_moe_backend(
def convert_to_unquantized_kernel_format(
unquantized_backend: UnquantizedMoeBackend,
layer: Module,
moe_config: FusedMoEConfig,
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -264,13 +268,13 @@ def convert_to_unquantized_kernel_format(
w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight)
elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_CUTLASS:
if layer.moe_config.is_act_and_mul:
if moe_config.is_act_and_mul:
# Swap halves to arrange as [w3; w1] (kernel expectation)
# Non-gated MoE: w13 is a single projection, no need to swap.
w13_weight = swap_w13_to_w31(w13_weight)
elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
is_act_and_mul = layer.moe_config.is_act_and_mul
is_act_and_mul = moe_config.is_act_and_mul
if not is_act_and_mul:
# Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK
# weight layout uses block_k=128). Pad along the intermediate dim when
@@ -278,7 +282,7 @@ def convert_to_unquantized_kernel_format(
w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi(
w13_weight, w2_weight, is_act_and_mul, min_alignment=128
)
layer.moe_config.intermediate_size_per_partition = padded_intermediate
moe_config.intermediate_size_per_partition = padded_intermediate
_cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
w13_weight, w2_weight = convert_moe_weights_to_flashinfer_trtllm_block_layout(
@@ -333,3 +337,71 @@ def make_unquantized_moe_kernel(
)
return kernel
# ---------------------------------------------------------------------------
# Class-based view (first PR of the #37753 series; see oracle/base.py).
# Methods delegate to the module-level functions above so behaviour is
# bit-identical with pre-class code.
# ---------------------------------------------------------------------------
class UnquantizedMoEKernelOracle(MoEKernelOracle[UnquantizedMoeBackend]):
"""Class-based view of the unquantized MoE kernel oracle.
Each method delegates to its module-level counterpart so that
instantiating and calling this class is bit-identical to calling
the standalone functions. Follow-up PRs may move logic from the
module-level functions into these methods.
"""
def backend_enum_cls(self) -> type[UnquantizedMoeBackend]:
return UnquantizedMoeBackend
def get_priority_backends(
self, moe_config: FusedMoEConfig
) -> list[UnquantizedMoeBackend]:
return _get_priority_backends(moe_config)
def backend_to_kernel_cls(
self, backend: UnquantizedMoeBackend
) -> type[mk.FusedMoEExperts]:
return backend_to_kernel_cls(backend)
def map_backend(self, runner_backend: MoEBackend) -> UnquantizedMoeBackend:
return map_unquantized_backend(runner_backend)
def select_backend(
self,
moe_config: FusedMoEConfig,
weight_key: "QuantKey | None" = None,
activation_key: "QuantKey | None" = None,
) -> tuple[UnquantizedMoeBackend, type[mk.FusedMoEExperts] | None]:
assert weight_key is None and activation_key is None, (
"Weights and activations will never be quantized for "
"UnquantizedMoEKernelOracle"
)
return select_unquantized_moe_backend(moe_config)
def convert_to_kernel_format(
self,
backend: UnquantizedMoeBackend,
moe_config: FusedMoEConfig,
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
return convert_to_unquantized_kernel_format(
backend, moe_config, w13_weight, w2_weight
)
def make_kernel(
self,
quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
backend: UnquantizedMoeBackend,
experts_cls: type[mk.FusedMoEExperts],
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEKernel:
return make_unquantized_moe_kernel(
quant_config, moe_config, backend, experts_cls, routing_tables
)
@@ -161,7 +161,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
# Shuffle weights to runtime format.
w13_new, w2_new = convert_to_unquantized_kernel_format(
self.unquantized_backend,
layer=layer,
moe_config=layer.moe_config,
w13_weight=w13,
w2_weight=w2,
)