[MoE] [MoE Refactor] Add moe kernel oracle abc 37753 (#43461)
Signed-off-by: Qiuyang Yue <yueqiuyang1389@gmail.com> Signed-off-by: qyYue1389 <yueqiuyang1389@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
2365b7a8e7
commit
9bfd878a48
@@ -0,0 +1,51 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the MoEKernelOracle ABC introduced in PR series for #37753.
|
||||
|
||||
This file contains a single canonical demonstration that
|
||||
`UnquantizedMoEKernelOracle` methods delegate one-to-one to the
|
||||
existing module-level functions in `oracle/unquantized.py`. Each method
|
||||
on `UnquantizedMoEKernelOracle` follows the same `return module_fn(args)`
|
||||
pattern, so verifying delegation for one method (`make_kernel`) gives
|
||||
high confidence in the rest.
|
||||
"""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
|
||||
from vllm.model_executor.layers.fused_moe.oracle import UnquantizedMoEKernelOracle
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
)
|
||||
|
||||
|
||||
class TestUnquantizedDelegation:
|
||||
"""UnquantizedMoEKernelOracle methods must delegate to the existing
|
||||
module-level functions; behaviour is bit-identical."""
|
||||
|
||||
def test_make_kernel_delegates(self) -> None:
|
||||
quant_config = object()
|
||||
moe_config = object()
|
||||
experts_cls = TritonExperts
|
||||
sentinel_kernel = object()
|
||||
|
||||
with patch(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized."
|
||||
"make_unquantized_moe_kernel",
|
||||
return_value=sentinel_kernel,
|
||||
) as mocked:
|
||||
out = UnquantizedMoEKernelOracle().make_kernel(
|
||||
quant_config,
|
||||
moe_config,
|
||||
UnquantizedMoeBackend.TRITON,
|
||||
experts_cls,
|
||||
)
|
||||
|
||||
mocked.assert_called_once_with(
|
||||
quant_config,
|
||||
moe_config,
|
||||
UnquantizedMoeBackend.TRITON,
|
||||
experts_cls,
|
||||
None, # routing_tables default
|
||||
)
|
||||
assert out is sentinel_kernel
|
||||
@@ -1,2 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.oracle.base import MoEKernelOracle
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoEKernelOracle,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MoEKernelOracle",
|
||||
"UnquantizedMoEKernelOracle",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Abstract base class for MoE kernel oracles.
|
||||
|
||||
Each MoE oracle (unquantized / fp8 / nvfp4 / mxfp4 / mxfp8 / int8 /
|
||||
int_wna16) is responsible for selecting the right MoE kernel backend for a
|
||||
given (model, hardware, deployment-config) tuple. The current
|
||||
implementation expresses this responsibility as module-level functions
|
||||
that follow an informal convention.
|
||||
|
||||
This module declares the abstract contract; concrete oracles inherit from
|
||||
`MoEKernelOracle` and provide the platform-specific behaviour.
|
||||
|
||||
This is the first PR in the series suggested by @robertgshaw2-redhat in
|
||||
PR #37776 (see issue #37753). It intentionally only introduces the ABC;
|
||||
follow-up PRs migrate each oracle to inherit from it. The single concrete
|
||||
subclass shipped here (`UnquantizedMoEKernelOracle`) delegates to the
|
||||
existing module-level functions to keep behaviour bit-identical with
|
||||
pre-class code.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from enum import Enum
|
||||
from typing import TYPE_CHECKING, Generic, TypeVar
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
BackendT = TypeVar("BackendT", bound=Enum)
|
||||
|
||||
|
||||
class MoEKernelOracle(ABC, Generic[BackendT]):
|
||||
"""Abstract base for MoE kernel-selection oracles.
|
||||
|
||||
Concrete oracles MUST implement: `backend_enum_cls`,
|
||||
`get_priority_backends`, `backend_to_kernel_cls`, `map_backend`,
|
||||
`select_backend`, `make_kernel`.
|
||||
|
||||
Concrete oracles MAY override: `convert_to_kernel_format`,
|
||||
`make_quant_config`. The base class provides default implementations
|
||||
that are appropriate for oracles which do not need them
|
||||
(e.g. `make_quant_config` raises on the unquantized oracle).
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def backend_enum_cls(self) -> type[BackendT]:
|
||||
"""Return the concrete `Enum` class enumerating this oracle's
|
||||
backends (e.g. `UnquantizedMoeBackend`, `Fp8MoeBackend`)."""
|
||||
|
||||
@abstractmethod
|
||||
def get_priority_backends(self, moe_config: FusedMoEConfig) -> list[BackendT]:
|
||||
"""Return platform-appropriate backends in priority order for
|
||||
this `moe_config`."""
|
||||
|
||||
@abstractmethod
|
||||
def backend_to_kernel_cls(self, backend: BackendT) -> type[mk.FusedMoEExperts]:
|
||||
"""Map a backend enum value to its concrete `FusedMoEExperts`
|
||||
subclass."""
|
||||
|
||||
@abstractmethod
|
||||
def map_backend(self, runner_backend: MoEBackend) -> BackendT:
|
||||
"""Map a user-facing `MoEBackend` (from the runner config) to
|
||||
this oracle's enum."""
|
||||
|
||||
@abstractmethod
|
||||
def select_backend(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
weight_key: "QuantKey | None" = None,
|
||||
activation_key: "QuantKey | None" = None,
|
||||
) -> tuple[BackendT, type[mk.FusedMoEExperts] | None]:
|
||||
"""Primary entry point: choose the best supported backend for
|
||||
the given `moe_config`.
|
||||
|
||||
`weight_key` / `activation_key` carry the quantization scheme of
|
||||
the weights and activations and are consumed by quantized oracles
|
||||
(fp8, nvfp4, int8, ...) to disambiguate backends. The unquantized
|
||||
oracle ignores them. Subclasses with additional selection inputs
|
||||
(e.g. int_wna16 needs `weight_bits`, fp8 needs
|
||||
`allow_vllm_cutlass`) widen the signature in their override; a
|
||||
per-oracle config object is the longer-term target tracked in
|
||||
the #37753 follow-up PRs.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def make_kernel(
|
||||
self,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
moe_config: FusedMoEConfig,
|
||||
backend: BackendT,
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
"""Construct the `FusedMoEKernel` (Prepare/Finalize + Experts
|
||||
combinator) for the chosen backend."""
|
||||
|
||||
def convert_to_kernel_format(
|
||||
self,
|
||||
backend: BackendT,
|
||||
moe_config: FusedMoEConfig,
|
||||
w13_weight: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Shuffle weights into the layout expected by `backend`.
|
||||
|
||||
Default implementation returns the inputs unchanged. Oracles
|
||||
whose backends need weight permutation should override this
|
||||
(e.g. `UnquantizedMoEKernelOracle` handles AITER and FlashInfer
|
||||
layouts).
|
||||
|
||||
`moe_config` carries MoE-layer state (e.g. `is_act_and_mul`)
|
||||
that the conversion needs without coupling the oracle to a
|
||||
`Module` reference. Quantized oracles whose conversion
|
||||
additionally needs scales / zero-points / block shapes will
|
||||
override with a wider signature (and ultimately a per-oracle
|
||||
config object — tracked in the #37753 follow-up PRs).
|
||||
"""
|
||||
return w13_weight, w2_weight
|
||||
|
||||
def make_quant_config(self, *args, **kwargs) -> FusedMoEQuantConfig:
|
||||
"""Build a `FusedMoEQuantConfig` for this oracle.
|
||||
|
||||
Quantized oracles (fp8, nvfp4, mxfp4, ...) override this with
|
||||
the appropriate signature for their quantization scheme.
|
||||
Unquantized oracles inherit the default, which raises because
|
||||
there is no quantization-specific config to build.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} does not implement make_quant_config; "
|
||||
"this oracle has no quantization-specific config to build."
|
||||
)
|
||||
@@ -2,9 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from enum import Enum
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
from torch.nn import Module
|
||||
|
||||
import vllm.envs as envs
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.base import MoEKernelOracle
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
align_moe_weights_for_fi,
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout,
|
||||
@@ -25,6 +26,9 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -256,7 +260,7 @@ def select_unquantized_moe_backend(
|
||||
|
||||
def convert_to_unquantized_kernel_format(
|
||||
unquantized_backend: UnquantizedMoeBackend,
|
||||
layer: Module,
|
||||
moe_config: FusedMoEConfig,
|
||||
w13_weight: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
@@ -264,13 +268,13 @@ def convert_to_unquantized_kernel_format(
|
||||
w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight)
|
||||
|
||||
elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_CUTLASS:
|
||||
if layer.moe_config.is_act_and_mul:
|
||||
if moe_config.is_act_and_mul:
|
||||
# Swap halves to arrange as [w3; w1] (kernel expectation)
|
||||
# Non-gated MoE: w13 is a single projection, no need to swap.
|
||||
w13_weight = swap_w13_to_w31(w13_weight)
|
||||
|
||||
elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
|
||||
is_act_and_mul = layer.moe_config.is_act_and_mul
|
||||
is_act_and_mul = moe_config.is_act_and_mul
|
||||
if not is_act_and_mul:
|
||||
# Kernel requires intermediate_size_per_partition % 128 == 0 (BlockMajorK
|
||||
# weight layout uses block_k=128). Pad along the intermediate dim when
|
||||
@@ -278,7 +282,7 @@ def convert_to_unquantized_kernel_format(
|
||||
w13_weight, w2_weight, padded_intermediate = align_moe_weights_for_fi(
|
||||
w13_weight, w2_weight, is_act_and_mul, min_alignment=128
|
||||
)
|
||||
layer.moe_config.intermediate_size_per_partition = padded_intermediate
|
||||
moe_config.intermediate_size_per_partition = padded_intermediate
|
||||
|
||||
_cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
|
||||
w13_weight, w2_weight = convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
@@ -333,3 +337,71 @@ def make_unquantized_moe_kernel(
|
||||
)
|
||||
|
||||
return kernel
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Class-based view (first PR of the #37753 series; see oracle/base.py).
|
||||
# Methods delegate to the module-level functions above so behaviour is
|
||||
# bit-identical with pre-class code.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class UnquantizedMoEKernelOracle(MoEKernelOracle[UnquantizedMoeBackend]):
|
||||
"""Class-based view of the unquantized MoE kernel oracle.
|
||||
|
||||
Each method delegates to its module-level counterpart so that
|
||||
instantiating and calling this class is bit-identical to calling
|
||||
the standalone functions. Follow-up PRs may move logic from the
|
||||
module-level functions into these methods.
|
||||
"""
|
||||
|
||||
def backend_enum_cls(self) -> type[UnquantizedMoeBackend]:
|
||||
return UnquantizedMoeBackend
|
||||
|
||||
def get_priority_backends(
|
||||
self, moe_config: FusedMoEConfig
|
||||
) -> list[UnquantizedMoeBackend]:
|
||||
return _get_priority_backends(moe_config)
|
||||
|
||||
def backend_to_kernel_cls(
|
||||
self, backend: UnquantizedMoeBackend
|
||||
) -> type[mk.FusedMoEExperts]:
|
||||
return backend_to_kernel_cls(backend)
|
||||
|
||||
def map_backend(self, runner_backend: MoEBackend) -> UnquantizedMoeBackend:
|
||||
return map_unquantized_backend(runner_backend)
|
||||
|
||||
def select_backend(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
weight_key: "QuantKey | None" = None,
|
||||
activation_key: "QuantKey | None" = None,
|
||||
) -> tuple[UnquantizedMoeBackend, type[mk.FusedMoEExperts] | None]:
|
||||
assert weight_key is None and activation_key is None, (
|
||||
"Weights and activations will never be quantized for "
|
||||
"UnquantizedMoEKernelOracle"
|
||||
)
|
||||
return select_unquantized_moe_backend(moe_config)
|
||||
|
||||
def convert_to_kernel_format(
|
||||
self,
|
||||
backend: UnquantizedMoeBackend,
|
||||
moe_config: FusedMoEConfig,
|
||||
w13_weight: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return convert_to_unquantized_kernel_format(
|
||||
backend, moe_config, w13_weight, w2_weight
|
||||
)
|
||||
|
||||
def make_kernel(
|
||||
self,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
moe_config: FusedMoEConfig,
|
||||
backend: UnquantizedMoeBackend,
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
return make_unquantized_moe_kernel(
|
||||
quant_config, moe_config, backend, experts_cls, routing_tables
|
||||
)
|
||||
|
||||
@@ -161,7 +161,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
# Shuffle weights to runtime format.
|
||||
w13_new, w2_new = convert_to_unquantized_kernel_format(
|
||||
self.unquantized_backend,
|
||||
layer=layer,
|
||||
moe_config=layer.moe_config,
|
||||
w13_weight=w13,
|
||||
w2_weight=w2,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user