From 1b57eb41f24166d0bee2031c53847b188089d77d Mon Sep 17 00:00:00 2001 From: bnellnm <49004751+bnellnm@users.noreply.github.com> Date: Sun, 10 May 2026 19:54:33 -0400 Subject: [PATCH] [MoE] Move various experts classes to fused_moe/experts/ (#41979) Signed-off-by: Jackmin801 Signed-off-by: Robert Shaw Signed-off-by: Jackmin801 <56836461+Jackmin801@users.noreply.github.com> Signed-off-by: Bill Nell Co-authored-by: Jackmin801 Co-authored-by: Claude Co-authored-by: Robert Shaw Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Jackmin801 <56836461+Jackmin801@users.noreply.github.com> --- csrc/quantization/gguf/moe.cuh | 2 +- docs/design/custom_op.md | 2 +- docs/design/moe_kernel_features.md | 6 +- .../moe/modular_kernel_tools/mk_objects.py | 4 +- tests/kernels/moe/test_flashinfer.py | 6 +- tests/kernels/moe/test_flashinfer_moe.py | 2 +- .../moe/test_marlin_vs_trtllm_mxint4.py | 2 +- tests/kernels/moe/test_moe.py | 2 +- tests/kernels/moe/test_rocm_aiter_topk.py | 2 +- .../kernels/moe/test_triton_moe_no_act_mul.py | 6 +- .../moe/test_unquantized_backend_selection.py | 2 +- tests/kernels/moe/utils.py | 4 +- .../layers/fused_moe/__init__.py | 12 +- .../{ => experts}/flashinfer_cutlass_moe.py | 0 .../marlin_moe.py} | 3 +- .../fused_moe/experts/nvfp4_emulation_moe.py | 2 +- .../fused_moe/experts/ocp_mx_emulation_moe.py | 2 +- .../rocm_aiter_moe.py} | 0 .../layers/fused_moe/experts/triton_moe.py | 522 ++++++++++++++++++ .../layers/fused_moe/fused_moe.py | 494 ----------------- vllm/model_executor/layers/fused_moe/layer.py | 6 +- .../layers/fused_moe/oracle/fp8.py | 8 +- .../layers/fused_moe/oracle/int8.py | 2 +- .../layers/fused_moe/oracle/int_wna16.py | 6 +- .../layers/fused_moe/oracle/mxfp4.py | 8 +- .../layers/fused_moe/oracle/nvfp4.py | 4 +- .../layers/fused_moe/oracle/unquantized.py | 8 +- .../fused_moe/router/grouped_topk_router.py | 2 +- .../layers/fused_moe/triton_cutlass_moe.py | 2 +- .../layers/fused_moe/triton_deep_gemm_moe.py | 2 +- .../layers/quantization/awq_marlin.py | 4 +- .../compressed_tensors_moe_w4a4_mxfp4.py | 2 +- .../compressed_tensors_moe_wna16_marlin.py | 2 +- .../layers/quantization/quark/quark_moe.py | 8 +- 34 files changed, 586 insertions(+), 553 deletions(-) rename vllm/model_executor/layers/fused_moe/{ => experts}/flashinfer_cutlass_moe.py (100%) rename vllm/model_executor/layers/fused_moe/{fused_marlin_moe.py => experts/marlin_moe.py} (99%) rename vllm/model_executor/layers/fused_moe/{rocm_aiter_fused_moe.py => experts/rocm_aiter_moe.py} (100%) create mode 100644 vllm/model_executor/layers/fused_moe/experts/triton_moe.py diff --git a/csrc/quantization/gguf/moe.cuh b/csrc/quantization/gguf/moe.cuh index df9b84abcc..a2f9f46c8f 100644 --- a/csrc/quantization/gguf/moe.cuh +++ b/csrc/quantization/gguf/moe.cuh @@ -1,7 +1,7 @@ #include /* Adapted from ./csrc/quantization/gguf/mmq.cuh - based on ./vllm/model_executor/layers/fused_moe/fused_moe.py */ + based on ./vllm/model_executor/layers/fused_moe/experts/triton_moe.py */ template 1 | G,A,T | silu, gelu,
swigluoai,
silu_no_mul,
gelu_no_mul | Y | Y | [`fused_experts`][vllm.model_executor.layers.fused_moe.fused_moe.fused_experts],
[`TritonExperts`][vllm.model_executor.layers.fused_moe.fused_moe.TritonExperts] | +| triton | standard | all1 | G,A,T | silu, gelu,
swigluoai,
silu_no_mul,
gelu_no_mul | Y | Y | [`fused_experts`][vllm.model_executor.layers.fused_moe.fused_moe.fused_experts],
[`TritonExperts`][vllm.model_executor.layers.fused_moe.experts.triton_moe.TritonExperts] | | triton (batched) | batched | all1 | G,A,T | silu, gelu | 6 | Y | [`BatchedTritonExperts`][vllm.model_executor.layers.fused_moe.fused_batched_moe.BatchedTritonExperts] | | deep gemm | standard,
batched | fp8 | G(128),A,T | silu, gelu | 6 | Y |
[`DeepGemmExperts`][vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe.DeepGemmExperts],
[`BatchedDeepGemmExperts`][vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe.BatchedDeepGemmExperts] | | cutlass_fp4 | standard,
batched | nvfp4 | A,T | silu | Y | Y | [`CutlassExpertsFp4`][vllm.model_executor.layers.fused_moe.experts.cutlass_moe.CutlassExpertsFp4] | | cutlass_fp8 | standard,
batched | fp8 | A,T | silu, gelu | Y | Y | [`CutlassExpertsFp8`][vllm.model_executor.layers.fused_moe.experts.cutlass_moe.CutlassExpertsFp8],
[`CutlasBatchedExpertsFp8`][vllm.model_executor.layers.fused_moe.experts.cutlass_moe.CutlassBatchedExpertsFp8] | -| flashinfer | standard | nvfp4,
fp8 | T | 5 | N | Y | [`FlashInferExperts`][vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe.FlashInferExperts] | +| flashinfer | standard | nvfp4,
fp8 | T | 5 | N | Y | [`FlashInferExperts`][vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe.FlashInferExperts] | | gpt oss triton | standard | N/A | N/A | 5 | Y | Y | [`triton_kernel_fused_experts`][vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe.triton_kernel_fused_experts],
[`OAITritonExperts`][vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe.OAITritonExperts] | -| marlin | standard,
batched | 3 / N/A | 3 / N/A | silu,
swigluoai | Y | Y | [`fused_marlin_moe`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.fused_marlin_moe],
[`MarlinExperts`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.MarlinExperts],
[`BatchedMarlinExperts`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.BatchedMarlinExperts] | +| marlin | standard,
batched | 3 / N/A | 3 / N/A | silu,
swigluoai | Y | Y | [`fused_marlin_moe`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.fused_marlin_moe],
[`MarlinExperts`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.MarlinExperts],
[`BatchedMarlinExperts`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.BatchedMarlinExperts] | | trtllm | standard | mxfp4,
nvfp4 | G(16),G(32) | 5 | N | Y | [`TrtLlmMxfp4ExpertsMonolithic`][vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe.TrtLlmMxfp4ExpertsMonolithic],
[`TrtLlmMxfp4ExpertsModular`][vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe.TrtLlmMxfp4ExpertsModular],
[`TrtLlmNvFp4ExpertsMonolithic`][vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe.TrtLlmNvFp4ExpertsMonolithic],
[`TrtLlmNvfp4ExpertsModular`][vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe.TrtLlmNvFp4ExpertsModular] | | rocm aiter moe | standard | mxfp4,
fp8 | G(32),G(128),A,T | silu, gelu,
swigluoai | Y | N | `rocm_aiter_fused_experts`,
`AiterExperts` | | cpu_fused_moe | standard | N/A | N/A | silu | N | N | [`CPUFusedMOE`][vllm.model_executor.layers.fused_moe.cpu_fused_moe.CPUFusedMOE] | diff --git a/tests/kernels/moe/modular_kernel_tools/mk_objects.py b/tests/kernels/moe/modular_kernel_tools/mk_objects.py index 812164ea28..111975cf6b 100644 --- a/tests/kernels/moe/modular_kernel_tools/mk_objects.py +++ b/tests/kernels/moe/modular_kernel_tools/mk_objects.py @@ -237,7 +237,7 @@ if has_mori(): ) if has_flashinfer_cutlass_fused_moe() and current_platform.has_device_capability(100): - from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import ( + from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( FlashInferExperts, ) from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_two_sided import ( # noqa: E501 @@ -298,7 +298,7 @@ if has_flashinfer_cutlass_fused_moe() and current_platform.has_device_capability ) if has_aiter(): - from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( AiterExperts, ) diff --git a/tests/kernels/moe/test_flashinfer.py b/tests/kernels/moe/test_flashinfer.py index dad25bc319..9e35be0db8 100644 --- a/tests/kernels/moe/test_flashinfer.py +++ b/tests/kernels/moe/test_flashinfer.py @@ -18,12 +18,12 @@ from vllm.model_executor.layers.fused_moe.config import ( RoutingMethodType, fp8_w8a8_moe_quant_config, ) +from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( + FlashInferExperts, +) from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import ( TrtLlmFp8ExpertsMonolithic, ) -from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import ( - FlashInferExperts, -) from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( rotate_weights_for_fi_trtllm_fp8_per_tensor_moe, diff --git a/tests/kernels/moe/test_flashinfer_moe.py b/tests/kernels/moe/test_flashinfer_moe.py index d116a96f58..6ae67fa0e9 100644 --- a/tests/kernels/moe/test_flashinfer_moe.py +++ b/tests/kernels/moe/test_flashinfer_moe.py @@ -22,7 +22,7 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEParallelConfig, RoutingMethodType, ) -from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import ( +from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( FlashInferExperts, is_valid_flashinfer_cutlass_fused_moe, ) diff --git a/tests/kernels/moe/test_marlin_vs_trtllm_mxint4.py b/tests/kernels/moe/test_marlin_vs_trtllm_mxint4.py index 0ad2cd06ee..eaeca6a8a5 100644 --- a/tests/kernels/moe/test_marlin_vs_trtllm_mxint4.py +++ b/tests/kernels/moe/test_marlin_vs_trtllm_mxint4.py @@ -5,7 +5,7 @@ import pytest import torch -from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( +from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( fused_marlin_moe, ) from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import ( diff --git a/tests/kernels/moe/test_moe.py b/tests/kernels/moe/test_moe.py index ebc3256b54..5441062825 100644 --- a/tests/kernels/moe/test_moe.py +++ b/tests/kernels/moe/test_moe.py @@ -32,7 +32,7 @@ from vllm.model_executor.layers.fused_moe.config import ( int4_w4a16_moe_quant_config, int8_w8a16_moe_quant_config, ) -from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( +from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( batched_fused_marlin_moe, fused_marlin_moe, ) diff --git a/tests/kernels/moe/test_rocm_aiter_topk.py b/tests/kernels/moe/test_rocm_aiter_topk.py index b0ecc9ed71..11f9cf28ae 100644 --- a/tests/kernels/moe/test_rocm_aiter_topk.py +++ b/tests/kernels/moe/test_rocm_aiter_topk.py @@ -20,7 +20,7 @@ if not current_platform.is_rocm(): pytest.skip("This test can only run on ROCm.", allow_module_level=True) # this import statement is needed to ensure the ops are registered -import vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe # noqa: F401 +import vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe # noqa: F401 # need to import once to ensure the ops are registered # Check if aiter package is installed diff --git a/tests/kernels/moe/test_triton_moe_no_act_mul.py b/tests/kernels/moe/test_triton_moe_no_act_mul.py index 1dfac3cf0f..9d16ae5b63 100644 --- a/tests/kernels/moe/test_triton_moe_no_act_mul.py +++ b/tests/kernels/moe/test_triton_moe_no_act_mul.py @@ -15,7 +15,7 @@ from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.config import ( FUSED_MOE_UNQUANTIZED_CONFIG, ) -from vllm.model_executor.layers.fused_moe.fused_moe import TritonExperts +from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts from vllm.platforms import current_platform # Test parameters @@ -151,7 +151,7 @@ def test_triton_experts_no_mul_activation( @torch.inference_mode() def test_workspace_shapes_no_mul_vs_gated(): """Test that workspace shapes differ correctly between gated and non-gated.""" - from vllm.model_executor.layers.fused_moe.fused_moe import TritonExperts + from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts M, N, K, topk = 64, 256, 128, 2 @@ -192,7 +192,7 @@ def test_workspace_shapes_no_mul_vs_gated(): @torch.inference_mode() def test_adjust_n_for_activation(): """Test the adjust_N_for_activation method.""" - from vllm.model_executor.layers.fused_moe.fused_moe import TritonExperts + from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts experts = TritonExperts( moe_config=make_dummy_moe_config(), diff --git a/tests/kernels/moe/test_unquantized_backend_selection.py b/tests/kernels/moe/test_unquantized_backend_selection.py index 2c4cd7b94e..bc322aed39 100644 --- a/tests/kernels/moe/test_unquantized_backend_selection.py +++ b/tests/kernels/moe/test_unquantized_backend_selection.py @@ -158,7 +158,7 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm, monkeyp return_value=(False, None), ) @patch( - "vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe.FlashInferExperts.is_supported_config", + "vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe.FlashInferExperts.is_supported_config", return_value=(True, None), ) @pytest.mark.skipif( diff --git a/tests/kernels/moe/utils.py b/tests/kernels/moe/utils.py index a5d6afc489..de3a456fb4 100644 --- a/tests/kernels/moe/utils.py +++ b/tests/kernels/moe/utils.py @@ -17,12 +17,14 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEQuantConfig, RoutingMethodType, ) +from vllm.model_executor.layers.fused_moe.experts.triton_moe import ( + TritonExperts, +) from vllm.model_executor.layers.fused_moe.fused_batched_moe import ( BatchedTritonExperts, NaiveBatchedExperts, ) from vllm.model_executor.layers.fused_moe.fused_moe import ( - TritonExperts, fused_experts, ) from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel diff --git a/vllm/model_executor/layers/fused_moe/__init__.py b/vllm/model_executor/layers/fused_moe/__init__.py index 75a9faddc1..aeb704e18d 100644 --- a/vllm/model_executor/layers/fused_moe/__init__.py +++ b/vllm/model_executor/layers/fused_moe/__init__.py @@ -85,6 +85,13 @@ if HAS_TRITON: from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import ( DeepGemmExperts, ) + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + AiterExperts, + ) + from vllm.model_executor.layers.fused_moe.experts.triton_moe import ( + TritonExperts, + TritonWNA16Experts, + ) from vllm.model_executor.layers.fused_moe.experts.xpu_moe import ( XPUExperts, XPUExpertsFp8, @@ -94,14 +101,9 @@ if HAS_TRITON: BatchedTritonExperts, ) from vllm.model_executor.layers.fused_moe.fused_moe import ( - TritonExperts, - TritonWNA16Experts, fused_experts, get_config_file_name, ) - from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( - AiterExperts, - ) from vllm.model_executor.layers.fused_moe.router.fused_topk_router import ( fused_topk, ) diff --git a/vllm/model_executor/layers/fused_moe/flashinfer_cutlass_moe.py b/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py similarity index 100% rename from vllm/model_executor/layers/fused_moe/flashinfer_cutlass_moe.py rename to vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py diff --git a/vllm/model_executor/layers/fused_moe/fused_marlin_moe.py b/vllm/model_executor/layers/fused_moe/experts/marlin_moe.py similarity index 99% rename from vllm/model_executor/layers/fused_moe/fused_marlin_moe.py rename to vllm/model_executor/layers/fused_moe/experts/marlin_moe.py index 3487ac1766..621b33dd8f 100644 --- a/vllm/model_executor/layers/fused_moe/fused_marlin_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/marlin_moe.py @@ -29,6 +29,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( from vllm.model_executor.layers.fused_moe.utils import ( _resize_cache, disable_inplace, + swiglu_limit_func, ) from vllm.model_executor.layers.quantization.utils.marlin_utils import ( get_marlin_input_dtype, @@ -50,8 +51,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( from vllm.platforms import current_platform from vllm.scalar_type import ScalarType, scalar_types -from .utils import swiglu_limit_func - def _fused_marlin_moe( hidden_states: torch.Tensor, diff --git a/vllm/model_executor/layers/fused_moe/experts/nvfp4_emulation_moe.py b/vllm/model_executor/layers/fused_moe/experts/nvfp4_emulation_moe.py index f1a0ee7ac5..de5b45ccb8 100644 --- a/vllm/model_executor/layers/fused_moe/experts/nvfp4_emulation_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/nvfp4_emulation_moe.py @@ -20,7 +20,7 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, ) -from vllm.model_executor.layers.fused_moe.fused_moe import TritonExperts +from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import ( dequantize_to_dtype, diff --git a/vllm/model_executor/layers/fused_moe/experts/ocp_mx_emulation_moe.py b/vllm/model_executor/layers/fused_moe/experts/ocp_mx_emulation_moe.py index 9fb163ef42..feb8c2ea76 100644 --- a/vllm/model_executor/layers/fused_moe/experts/ocp_mx_emulation_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/ocp_mx_emulation_moe.py @@ -20,7 +20,7 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, ) -from vllm.model_executor.layers.fused_moe.fused_moe import TritonExperts +from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input from vllm.model_executor.layers.quantization.utils.mxfp4_utils import dequant_mxfp4 from vllm.model_executor.layers.quantization.utils.mxfp6_utils import dequant_mxfp6 diff --git a/vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py similarity index 100% rename from vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py rename to vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py diff --git a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py new file mode 100644 index 0000000000..e99f307ec0 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py @@ -0,0 +1,522 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Triton-based MoE expert implementations.""" + +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm import _custom_ops as ops +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEParallelConfig, + FusedMoEQuantConfig, +) +from vllm.model_executor.layers.fused_moe.fused_moe import ( + _prepare_expert_assignment, + invoke_fused_moe_triton_kernel, + invoke_fused_moe_wna16_triton_kernel, + try_get_optimal_moe_config, +) +from vllm.model_executor.layers.fused_moe.lora_experts_mixin import ( + LoRAExpertsMixin, +) +from vllm.model_executor.layers.fused_moe.moe_align_block_size import ( + moe_align_block_size, +) +from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( + TopKWeightAndReduceNoOP, +) +from vllm.model_executor.layers.fused_moe.utils import ( + _resize_cache, + moe_kernel_quantize_input, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kFp8Dynamic128Sym, + kFp8DynamicTensorSym, + kFp8DynamicTokenSym, + kFp8Static128BlockSym, + kFp8StaticChannelSym, + kFp8StaticTensorSym, + kInt8DynamicTokenSym, + kInt8StaticChannelSym, +) +from vllm.platforms import current_platform +from vllm.triton_utils import tl + + +class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): + """Triton-based fused MoE expert implementation.""" + + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + ): + # Whether quantized MOE runs natively, or through + # higher-precision + activation QDQ. + self.quantization_emulation = False + super().__init__(moe_config, quant_config) + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def _supports_current_device() -> bool: + return current_platform.is_cuda_alike() or current_platform.is_xpu() + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return True + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + # INT8 requires at least 7.5 (Turing). + device_supports_int8 = ( + current_platform.is_cuda() + and current_platform.has_device_capability((7, 5)) + ) + + supported: list[tuple[QuantKey | None, QuantKey | None]] = [(None, None)] + if device_supports_int8: + supported.append((kInt8StaticChannelSym, kInt8DynamicTokenSym)) + if current_platform.supports_fp8(): + supported += [ + (kFp8Static128BlockSym, kFp8Dynamic128Sym), + (kFp8StaticChannelSym, kFp8DynamicTokenSym), + (kFp8StaticTensorSym, kFp8DynamicTokenSym), + (kFp8StaticTensorSym, kFp8StaticTensorSym), + (kFp8StaticTensorSym, kFp8DynamicTensorSym), + ] + return (weight_key, activation_key) in supported + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return activation in [ + MoEActivation.SILU, + MoEActivation.GELU, + MoEActivation.GELU_TANH, + MoEActivation.SWIGLUOAI, + MoEActivation.SWIGLUSTEP, + MoEActivation.SILU_NO_MUL, + MoEActivation.GELU_NO_MUL, + MoEActivation.GELU_TANH_NO_MUL, + MoEActivation.RELU2_NO_MUL, + ] + + @staticmethod + def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: + return not ( + moe_parallel_config.use_fi_nvl_two_sided_kernels + or moe_parallel_config.use_fi_nvl_one_sided_kernels + ) + + @staticmethod + def _supports_batch_invariance(): + return True + + def supports_expert_map(self) -> bool: + return True + + def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: + return TopKWeightAndReduceNoOP() + + def workspace_shapes( + self, + M: int, + N: int, + K: int, + topk: int, + global_num_experts: int, + local_num_experts: int, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + activation: MoEActivation, + ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: + activation_out_dim = self.adjust_N_for_activation(N, activation) + workspace1 = (M, topk, max(activation_out_dim, K)) + workspace2 = (M, topk, max(N, K)) + output = (M, K) + return (workspace1, workspace2, output) + + def apply( + self, + output: torch.Tensor, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, + workspace2: torch.Tensor, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + apply_router_weight_on_input: bool, + ): + # Check constraints. + if self.quant_config.use_int4_w4a16: + assert hidden_states.size(-1) // 2 == w1.size(2), "Hidden size mismatch" + else: + assert hidden_states.size(-1) == w1.size(2), ( + f"Hidden size mismatch {hidden_states.size(-1)} != {w1.size(2)}" + ) + + assert hidden_states.is_contiguous(), "Hidden_states must be contiguous" + assert hidden_states.dim() == 2 + assert w1.stride(-1) == 1, "Stride of last dimension must be 1" + assert w2.stride(-1) == 1, "Stride of last dimension must be 1" + assert hidden_states.dtype in [ + torch.float32, + torch.float16, + torch.bfloat16, + torch.float8_e4m3fn, + torch.float8_e4m3fnuz, + ] + + E, num_tokens, N, K, top_k_num = self.moe_problem_size( + hidden_states, w1, w2, topk_ids + ) + + if global_num_experts == -1: + global_num_experts = E + + config = try_get_optimal_moe_config( + w1.size(), + w2.size(), + top_k_num, + self.quant_config.config_name(hidden_states.dtype), + num_tokens, + block_shape=self.block_shape, + ) + + if hidden_states.dtype == torch.bfloat16: + compute_type = tl.bfloat16 + elif hidden_states.dtype == torch.float16: + compute_type = tl.float16 + elif hidden_states.dtype == torch.float32: + compute_type = tl.float32 + elif ( + hidden_states.dtype == torch.float8_e4m3fn + or hidden_states.dtype == torch.float8_e4m3fnuz + ): + compute_type = tl.bfloat16 + else: + raise ValueError(f"Unsupported compute_type: {hidden_states.dtype}") + + # Note that the output tensor might be in workspace1 + intermediate_cache1 = _resize_cache(workspace2, (num_tokens, top_k_num, N)) + cache2_dim = self.adjust_N_for_activation(N, activation) + intermediate_cache2 = _resize_cache( + workspace13, (num_tokens * top_k_num, cache2_dim) + ) + intermediate_cache3 = _resize_cache(workspace2, (num_tokens, top_k_num, K)) + + sorted_token_ids, expert_ids, num_tokens_post_padded = ( + _prepare_expert_assignment( + topk_ids, + config, + num_tokens, + top_k_num, + global_num_experts, + expert_map, + use_int8_w8a16=self.quant_config.use_int8_w8a16, + use_int4_w4a16=self.quant_config.use_int4_w4a16, + block_shape=self.block_shape, + ) + ) + + invoke_fused_moe_triton_kernel( + hidden_states, + w1, + intermediate_cache1, + a1q_scale, + self.w1_scale, + None, # topk_weights + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + False, # mul_routed_weights + top_k_num, + config, + compute_type=compute_type, + use_fp8_w8a8=self.quant_config.use_fp8_w8a8, + use_int8_w8a8=self.quant_config.use_int8_w8a8, + use_int8_w8a16=self.quant_config.use_int8_w8a16, + use_int4_w4a16=self.quant_config.use_int4_w4a16, + per_channel_quant=self.per_act_token_quant, + block_shape=self.block_shape, + B_bias=self.w1_bias, + ) + + # LoRA w13: applied to intermediate_cache1 before activation, using + # hidden_states as the lora_a input. moe_lora_align_block_size is + # called once here and results reused for the w2 LoRA below. + sorted_token_ids_lora = None + expert_ids_lora = None + num_tokens_post_padded_lora = None + token_lora_mapping = None + lora_context = self._lora_context + if lora_context is not None: + ( + sorted_token_ids_lora, + expert_ids_lora, + num_tokens_post_padded_lora, + token_lora_mapping, + ) = self.apply_w13_lora( + lora_context, + y=intermediate_cache1, + x=hidden_states, + topk_ids=topk_ids, + topk_weights=topk_weights, + expert_map=expert_map, + w1=w1, + w2=w2, + num_tokens=num_tokens, + top_k_num=top_k_num, + ) + + self.activation( + activation, intermediate_cache2, intermediate_cache1.view(-1, N) + ) + + a2q_scale: torch.Tensor | None = None + + qintermediate_cache2, a2q_scale = moe_kernel_quantize_input( + intermediate_cache2, + a2_scale, + self.quant_dtype, + self.per_act_token_quant, + self.block_shape, + quantization_emulation=self.quantization_emulation, + ) + + invoke_fused_moe_triton_kernel( + qintermediate_cache2, + w2, + intermediate_cache3, + a2q_scale, + self.w2_scale, + topk_weights, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + not apply_router_weight_on_input, + 1, + config, + compute_type=compute_type, + use_fp8_w8a8=self.quant_config.use_fp8_w8a8, + use_int8_w8a8=self.quant_config.use_int8_w8a8, + use_int8_w8a16=self.quant_config.use_int8_w8a16, + use_int4_w4a16=self.quant_config.use_int4_w4a16, + per_channel_quant=self.per_act_token_quant, + block_shape=self.block_shape, + B_bias=self.w2_bias, + ) + + # LoRA w2: applied to intermediate_cache3 before moe_sum, using the + # unquantized intermediate_cache2 as the lora_a input. Reuses the + # sorted_token_ids_lora computed above. + if lora_context is not None: + self.apply_w2_lora( + lora_context, + y=intermediate_cache3, + x=intermediate_cache2, + topk_weights=topk_weights, + sorted_token_ids_lora=sorted_token_ids_lora, + expert_ids_lora=expert_ids_lora, + num_tokens_post_padded_lora=num_tokens_post_padded_lora, + token_lora_mapping=token_lora_mapping, + num_tokens=num_tokens, + w1=w1, + w2=w2, + top_k_num=top_k_num, + ) + + # separate function is required for MoE + LoRA + self.moe_sum(intermediate_cache3, output) + + def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None: + ops.moe_sum(input, output) + + +class TritonWNA16Experts(TritonExperts): + @staticmethod + def _supports_current_device() -> bool: + raise NotImplementedError( + "TritonWNA16Experts is not yet used by an Oracle. " + "This method should not be called." + ) + + @staticmethod + def _supports_no_act_and_mul() -> bool: + raise NotImplementedError( + "TritonWNA16Experts is not yet used by an Oracle. " + "This method should not be called." + ) + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + raise NotImplementedError( + "TritonWNA16Experts is not yet used by an Oracle. " + "This method should not be called." + ) + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + raise NotImplementedError( + "TritonWNA16Experts is not yet used by an Oracle. " + "This method should not be called." + ) + + @staticmethod + def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: + raise NotImplementedError( + "TritonWNA16Experts is not yet used by an Oracle. " + "This method should not be called." + ) + + def apply( + self, + output: torch.Tensor, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, + workspace2: torch.Tensor, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + apply_router_weight_on_input: bool, + ): + # Check constraints. + if self.quant_config.use_int4_w4a16: + assert hidden_states.size(-1) // 2 == w1.size(2), "Hidden size mismatch" + else: + assert hidden_states.size(-1) == w1.size(2), ( + f"Hidden size mismatch {hidden_states.size(-1)} != {w1.size(2)}" + ) + + assert hidden_states.is_contiguous(), "Hidden_states must be contiguous" + assert hidden_states.dim() == 2 + assert w1.stride(-1) == 1, "Stride of last dimension must be 1" + assert w2.stride(-1) == 1, "Stride of last dimension must be 1" + assert hidden_states.dtype in [ + torch.float32, + torch.float16, + torch.bfloat16, + torch.float8_e4m3fn, + torch.float8_e4m3fnuz, + ] + + E, num_tokens, N, K, top_k_num = self.moe_problem_size( + hidden_states, w1, w2, topk_ids + ) + + if global_num_experts == -1: + global_num_experts = E + + config = try_get_optimal_moe_config( + w1.size(), + w2.size(), + top_k_num, + self.quant_config.config_name(hidden_states.dtype), + num_tokens, + block_shape=self.block_shape, + ) + + if hidden_states.dtype == torch.bfloat16: + compute_type = tl.bfloat16 + elif hidden_states.dtype == torch.float16: + compute_type = tl.float16 + elif hidden_states.dtype == torch.float32: + compute_type = tl.float32 + elif ( + hidden_states.dtype == torch.float8_e4m3fn + or hidden_states.dtype == torch.float8_e4m3fnuz + ): + compute_type = tl.bfloat16 + else: + raise ValueError(f"Unsupported compute_type: {hidden_states.dtype}") + + # Note that the output tensor might be in workspace1 + intermediate_cache1 = _resize_cache(workspace2, (num_tokens, top_k_num, N)) + activation_out_dim = self.adjust_N_for_activation(N, activation) + intermediate_cache2 = _resize_cache( + workspace13, (num_tokens * top_k_num, activation_out_dim) + ) + intermediate_cache3 = _resize_cache(workspace2, (num_tokens, top_k_num, K)) + + sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( + topk_ids, config["BLOCK_SIZE_M"], global_num_experts, expert_map + ) + + invoke_fused_moe_wna16_triton_kernel( + hidden_states, + w1, + intermediate_cache1, + self.w1_scale, + self.quant_config.w1_zp, + None, # topk_weights + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + False, # mul_routed_weights + top_k_num, + config, + compute_type=compute_type, + use_int8_w8a16=self.quant_config.use_int8_w8a16, + use_int4_w4a16=self.quant_config.use_int4_w4a16, + block_shape=self.block_shape, + ) + + self.activation( + activation, intermediate_cache2, intermediate_cache1.view(-1, N) + ) + + a2q_scale: torch.Tensor | None = None + + qintermediate_cache2, a2q_scale = moe_kernel_quantize_input( + intermediate_cache2, + a2_scale, + self.quant_dtype, + self.per_act_token_quant, + self.block_shape, + ) + + invoke_fused_moe_wna16_triton_kernel( + qintermediate_cache2, + w2, + intermediate_cache3, + self.w2_scale, + self.quant_config.w2_zp, + topk_weights, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + not apply_router_weight_on_input, + 1, + config, + compute_type=compute_type, + use_int8_w8a16=self.quant_config.use_int8_w8a16, + use_int4_w4a16=self.quant_config.use_int4_w4a16, + block_shape=self.block_shape, + ) + + # separate function is required for MoE + LoRA + self.moe_sum(intermediate_cache3, output) diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py index 1a65593425..8d974ea567 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -20,34 +20,16 @@ from vllm.model_executor.layers.fused_moe.activation import ( ) from vllm.model_executor.layers.fused_moe.config import ( FUSED_MOE_UNQUANTIZED_CONFIG, - FusedMoEConfig, - FusedMoEParallelConfig, FusedMoEQuantConfig, _get_config_dtype_str, ) -from vllm.model_executor.layers.fused_moe.lora_experts_mixin import LoRAExpertsMixin from vllm.model_executor.layers.fused_moe.moe_align_block_size import ( moe_align_block_size, ) -from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( - TopKWeightAndReduceNoOP, -) from vllm.model_executor.layers.fused_moe.utils import ( - _resize_cache, disable_inplace, moe_kernel_quantize_input, ) -from vllm.model_executor.layers.quantization.utils.quant_utils import ( - QuantKey, - kFp8Dynamic128Sym, - kFp8DynamicTensorSym, - kFp8DynamicTokenSym, - kFp8Static128BlockSym, - kFp8StaticChannelSym, - kFp8StaticTensorSym, - kInt8DynamicTokenSym, - kInt8StaticChannelSym, -) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.torch_utils import direct_register_custom_op @@ -1885,479 +1867,3 @@ def fused_experts_impl( ) return out_hidden_states - - -class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): - """Triton-based fused MoE expert implementation.""" - - def __init__( - self, - moe_config: FusedMoEConfig, - quant_config: FusedMoEQuantConfig, - ): - # Whether quantized MOE runs natively, or through - # higher-precision + activation QDQ. - self.quantization_emulation = False - super().__init__(moe_config, quant_config) - - @staticmethod - def activation_format() -> mk.FusedMoEActivationFormat: - return mk.FusedMoEActivationFormat.Standard - - @staticmethod - def _supports_current_device() -> bool: - return current_platform.is_cuda_alike() or current_platform.is_xpu() - - @staticmethod - def _supports_no_act_and_mul() -> bool: - return True - - @staticmethod - def _supports_quant_scheme( - weight_key: QuantKey | None, - activation_key: QuantKey | None, - ) -> bool: - # INT8 requires at least 7.5 (Turing). - device_supports_int8 = ( - current_platform.is_cuda() - and current_platform.has_device_capability((7, 5)) - ) - - supported: list[tuple[QuantKey | None, QuantKey | None]] = [(None, None)] - if device_supports_int8: - supported.append((kInt8StaticChannelSym, kInt8DynamicTokenSym)) - if current_platform.supports_fp8(): - supported += [ - (kFp8Static128BlockSym, kFp8Dynamic128Sym), - (kFp8StaticChannelSym, kFp8DynamicTokenSym), - (kFp8StaticTensorSym, kFp8DynamicTokenSym), - (kFp8StaticTensorSym, kFp8StaticTensorSym), - (kFp8StaticTensorSym, kFp8DynamicTensorSym), - ] - return (weight_key, activation_key) in supported - - @staticmethod - def _supports_activation(activation: MoEActivation) -> bool: - return activation in [ - MoEActivation.SILU, - MoEActivation.GELU, - MoEActivation.GELU_TANH, - MoEActivation.SWIGLUOAI, - MoEActivation.SWIGLUSTEP, - MoEActivation.SILU_NO_MUL, - MoEActivation.GELU_NO_MUL, - MoEActivation.GELU_TANH_NO_MUL, - MoEActivation.RELU2_NO_MUL, - ] - - @staticmethod - def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: - return not ( - moe_parallel_config.use_fi_nvl_two_sided_kernels - or moe_parallel_config.use_fi_nvl_one_sided_kernels - ) - - @staticmethod - def _supports_batch_invariance(): - return True - - def supports_expert_map(self) -> bool: - return True - - def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: - return TopKWeightAndReduceNoOP() - - def workspace_shapes( - self, - M: int, - N: int, - K: int, - topk: int, - global_num_experts: int, - local_num_experts: int, - expert_tokens_meta: mk.ExpertTokensMetadata | None, - activation: MoEActivation, - ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: - activation_out_dim = self.adjust_N_for_activation(N, activation) - workspace1 = (M, topk, max(activation_out_dim, K)) - workspace2 = (M, topk, max(N, K)) - output = (M, K) - return (workspace1, workspace2, output) - - def apply( - self, - output: torch.Tensor, - hidden_states: torch.Tensor, - w1: torch.Tensor, - w2: torch.Tensor, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - activation: MoEActivation, - global_num_experts: int, - expert_map: torch.Tensor | None, - a1q_scale: torch.Tensor | None, - a2_scale: torch.Tensor | None, - workspace13: torch.Tensor, - workspace2: torch.Tensor, - expert_tokens_meta: mk.ExpertTokensMetadata | None, - apply_router_weight_on_input: bool, - ): - # Check constraints. - if self.quant_config.use_int4_w4a16: - assert hidden_states.size(-1) // 2 == w1.size(2), "Hidden size mismatch" - else: - assert hidden_states.size(-1) == w1.size(2), ( - f"Hidden size mismatch {hidden_states.size(-1)} != {w1.size(2)}" - ) - - assert hidden_states.is_contiguous(), "Hidden_states must be contiguous" - assert hidden_states.dim() == 2 - assert w1.stride(-1) == 1, "Stride of last dimension must be 1" - assert w2.stride(-1) == 1, "Stride of last dimension must be 1" - assert hidden_states.dtype in [ - torch.float32, - torch.float16, - torch.bfloat16, - torch.float8_e4m3fn, - torch.float8_e4m3fnuz, - ] - - E, num_tokens, N, K, top_k_num = self.moe_problem_size( - hidden_states, w1, w2, topk_ids - ) - - if global_num_experts == -1: - global_num_experts = E - - config = try_get_optimal_moe_config( - w1.size(), - w2.size(), - top_k_num, - self.quant_config.config_name(hidden_states.dtype), - num_tokens, - block_shape=self.block_shape, - ) - - if hidden_states.dtype == torch.bfloat16: - compute_type = tl.bfloat16 - elif hidden_states.dtype == torch.float16: - compute_type = tl.float16 - elif hidden_states.dtype == torch.float32: - compute_type = tl.float32 - elif ( - hidden_states.dtype == torch.float8_e4m3fn - or hidden_states.dtype == torch.float8_e4m3fnuz - ): - compute_type = tl.bfloat16 - else: - raise ValueError(f"Unsupported compute_type: {hidden_states.dtype}") - - # Note that the output tensor might be in workspace1 - intermediate_cache1 = _resize_cache(workspace2, (num_tokens, top_k_num, N)) - cache2_dim = self.adjust_N_for_activation(N, activation) - intermediate_cache2 = _resize_cache( - workspace13, (num_tokens * top_k_num, cache2_dim) - ) - intermediate_cache3 = _resize_cache(workspace2, (num_tokens, top_k_num, K)) - - sorted_token_ids, expert_ids, num_tokens_post_padded = ( - _prepare_expert_assignment( - topk_ids, - config, - num_tokens, - top_k_num, - global_num_experts, - expert_map, - use_int8_w8a16=self.quant_config.use_int8_w8a16, - use_int4_w4a16=self.quant_config.use_int4_w4a16, - block_shape=self.block_shape, - ) - ) - - invoke_fused_moe_triton_kernel( - hidden_states, - w1, - intermediate_cache1, - a1q_scale, - self.w1_scale, - None, # topk_weights - sorted_token_ids, - expert_ids, - num_tokens_post_padded, - False, # mul_routed_weights - top_k_num, - config, - compute_type=compute_type, - use_fp8_w8a8=self.quant_config.use_fp8_w8a8, - use_int8_w8a8=self.quant_config.use_int8_w8a8, - use_int8_w8a16=self.quant_config.use_int8_w8a16, - use_int4_w4a16=self.quant_config.use_int4_w4a16, - per_channel_quant=self.per_act_token_quant, - block_shape=self.block_shape, - B_bias=self.w1_bias, - ) - - # LoRA w13: applied to intermediate_cache1 before activation, using - # hidden_states as the lora_a input. moe_lora_align_block_size is - # called once here and results reused for the w2 LoRA below. - sorted_token_ids_lora = None - expert_ids_lora = None - num_tokens_post_padded_lora = None - token_lora_mapping = None - lora_context = self._lora_context - if lora_context is not None: - ( - sorted_token_ids_lora, - expert_ids_lora, - num_tokens_post_padded_lora, - token_lora_mapping, - ) = self.apply_w13_lora( - lora_context, - y=intermediate_cache1, - x=hidden_states, - topk_ids=topk_ids, - topk_weights=topk_weights, - expert_map=expert_map, - w1=w1, - w2=w2, - num_tokens=num_tokens, - top_k_num=top_k_num, - ) - - self.activation( - activation, intermediate_cache2, intermediate_cache1.view(-1, N) - ) - - a2q_scale: torch.Tensor | None = None - - qintermediate_cache2, a2q_scale = moe_kernel_quantize_input( - intermediate_cache2, - a2_scale, - self.quant_dtype, - self.per_act_token_quant, - self.block_shape, - quantization_emulation=self.quantization_emulation, - ) - - invoke_fused_moe_triton_kernel( - qintermediate_cache2, - w2, - intermediate_cache3, - a2q_scale, - self.w2_scale, - topk_weights, - sorted_token_ids, - expert_ids, - num_tokens_post_padded, - not apply_router_weight_on_input, - 1, - config, - compute_type=compute_type, - use_fp8_w8a8=self.quant_config.use_fp8_w8a8, - use_int8_w8a8=self.quant_config.use_int8_w8a8, - use_int8_w8a16=self.quant_config.use_int8_w8a16, - use_int4_w4a16=self.quant_config.use_int4_w4a16, - per_channel_quant=self.per_act_token_quant, - block_shape=self.block_shape, - B_bias=self.w2_bias, - ) - - # LoRA w2: applied to intermediate_cache3 before moe_sum, using the - # unquantized intermediate_cache2 as the lora_a input. Reuses the - # sorted_token_ids_lora computed above. - if lora_context is not None: - self.apply_w2_lora( - lora_context, - y=intermediate_cache3, - x=intermediate_cache2, - topk_weights=topk_weights, - sorted_token_ids_lora=sorted_token_ids_lora, - expert_ids_lora=expert_ids_lora, - num_tokens_post_padded_lora=num_tokens_post_padded_lora, - token_lora_mapping=token_lora_mapping, - num_tokens=num_tokens, - w1=w1, - w2=w2, - top_k_num=top_k_num, - ) - - # separate function is required for MoE + LoRA - self.moe_sum(intermediate_cache3, output) - - def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None: - ops.moe_sum(input, output) - - -class TritonWNA16Experts(TritonExperts): - @staticmethod - def _supports_current_device() -> bool: - raise NotImplementedError( - "TritonWNA16Experts is not yet used by an Oracle. " - "This method should not be called." - ) - - @staticmethod - def _supports_no_act_and_mul() -> bool: - raise NotImplementedError( - "TritonWNA16Experts is not yet used by an Oracle. " - "This method should not be called." - ) - - @staticmethod - def _supports_quant_scheme( - weight_key: QuantKey | None, - activation_key: QuantKey | None, - ) -> bool: - raise NotImplementedError( - "TritonWNA16Experts is not yet used by an Oracle. " - "This method should not be called." - ) - - @staticmethod - def _supports_activation(activation: MoEActivation) -> bool: - raise NotImplementedError( - "TritonWNA16Experts is not yet used by an Oracle. " - "This method should not be called." - ) - - @staticmethod - def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: - raise NotImplementedError( - "TritonWNA16Experts is not yet used by an Oracle. " - "This method should not be called." - ) - - def apply( - self, - output: torch.Tensor, - hidden_states: torch.Tensor, - w1: torch.Tensor, - w2: torch.Tensor, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - activation: MoEActivation, - global_num_experts: int, - expert_map: torch.Tensor | None, - a1q_scale: torch.Tensor | None, - a2_scale: torch.Tensor | None, - workspace13: torch.Tensor, - workspace2: torch.Tensor, - expert_tokens_meta: mk.ExpertTokensMetadata | None, - apply_router_weight_on_input: bool, - ): - # Check constraints. - if self.quant_config.use_int4_w4a16: - assert hidden_states.size(-1) // 2 == w1.size(2), "Hidden size mismatch" - else: - assert hidden_states.size(-1) == w1.size(2), ( - f"Hidden size mismatch {hidden_states.size(-1)} != {w1.size(2)}" - ) - - assert hidden_states.is_contiguous(), "Hidden_states must be contiguous" - assert hidden_states.dim() == 2 - assert w1.stride(-1) == 1, "Stride of last dimension must be 1" - assert w2.stride(-1) == 1, "Stride of last dimension must be 1" - assert hidden_states.dtype in [ - torch.float32, - torch.float16, - torch.bfloat16, - torch.float8_e4m3fn, - torch.float8_e4m3fnuz, - ] - - E, num_tokens, N, K, top_k_num = self.moe_problem_size( - hidden_states, w1, w2, topk_ids - ) - - if global_num_experts == -1: - global_num_experts = E - - config = try_get_optimal_moe_config( - w1.size(), - w2.size(), - top_k_num, - self.quant_config.config_name(hidden_states.dtype), - num_tokens, - block_shape=self.block_shape, - ) - - if hidden_states.dtype == torch.bfloat16: - compute_type = tl.bfloat16 - elif hidden_states.dtype == torch.float16: - compute_type = tl.float16 - elif hidden_states.dtype == torch.float32: - compute_type = tl.float32 - elif ( - hidden_states.dtype == torch.float8_e4m3fn - or hidden_states.dtype == torch.float8_e4m3fnuz - ): - compute_type = tl.bfloat16 - else: - raise ValueError(f"Unsupported compute_type: {hidden_states.dtype}") - - # Note that the output tensor might be in workspace1 - intermediate_cache1 = _resize_cache(workspace2, (num_tokens, top_k_num, N)) - activation_out_dim = self.adjust_N_for_activation(N, activation) - intermediate_cache2 = _resize_cache( - workspace13, (num_tokens * top_k_num, activation_out_dim) - ) - intermediate_cache3 = _resize_cache(workspace2, (num_tokens, top_k_num, K)) - - sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( - topk_ids, config["BLOCK_SIZE_M"], global_num_experts, expert_map - ) - - invoke_fused_moe_wna16_triton_kernel( - hidden_states, - w1, - intermediate_cache1, - self.w1_scale, - self.quant_config.w1_zp, - None, # topk_weights - sorted_token_ids, - expert_ids, - num_tokens_post_padded, - False, # mul_routed_weights - top_k_num, - config, - compute_type=compute_type, - use_int8_w8a16=self.quant_config.use_int8_w8a16, - use_int4_w4a16=self.quant_config.use_int4_w4a16, - block_shape=self.block_shape, - ) - - self.activation( - activation, intermediate_cache2, intermediate_cache1.view(-1, N) - ) - - a2q_scale: torch.Tensor | None = None - - qintermediate_cache2, a2q_scale = moe_kernel_quantize_input( - intermediate_cache2, - a2_scale, - self.quant_dtype, - self.per_act_token_quant, - self.block_shape, - ) - - invoke_fused_moe_wna16_triton_kernel( - qintermediate_cache2, - w2, - intermediate_cache3, - self.w2_scale, - self.quant_config.w2_zp, - topk_weights, - sorted_token_ids, - expert_ids, - num_tokens_post_padded, - not apply_router_weight_on_input, - 1, - config, - compute_type=compute_type, - use_int8_w8a16=self.quant_config.use_int8_w8a16, - use_int4_w4a16=self.quant_config.use_int4_w4a16, - block_shape=self.block_shape, - ) - - # separate function is required for MoE + LoRA - self.moe_sum(intermediate_cache3, output) diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index acb601eff5..2eef89793a 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -26,15 +26,15 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEQuantConfig, RoutingMethodType, ) +from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + init_aiter_topK_meta_data, +) from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( FusedMoEMethodBase, ) from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import ( FusedMoEModularMethod, ) -from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( - init_aiter_topK_meta_data, -) from vllm.model_executor.layers.fused_moe.router.router_factory import ( create_fused_moe_router, ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index b65f8efa4a..1ce5a1e321 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -123,7 +123,7 @@ def backend_to_kernel_cls( return [TrtLlmFp8ExpertsMonolithic, TrtLlmFp8ExpertsModular] elif backend == Fp8MoeBackend.FLASHINFER_CUTLASS: - from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import ( + from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( # noqa: E501 FlashInferExperts, ) @@ -144,14 +144,14 @@ def backend_to_kernel_cls( return [BatchedDeepGemmExperts] elif backend == Fp8MoeBackend.MARLIN: - from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( + from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( MarlinExperts, ) return [MarlinExperts] elif backend == Fp8MoeBackend.TRITON: - from vllm.model_executor.layers.fused_moe.fused_moe import ( + from vllm.model_executor.layers.fused_moe.experts.triton_moe import ( TritonExperts, ) @@ -165,7 +165,7 @@ def backend_to_kernel_cls( return [BatchedTritonExperts] elif backend == Fp8MoeBackend.AITER: - from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( AiterExperts, ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/int8.py b/vllm/model_executor/layers/fused_moe/oracle/int8.py index ebdd20d54d..0c1d641a8e 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int8.py @@ -46,7 +46,7 @@ def backend_to_kernel_cls( backend: Int8MoeBackend, ) -> list[type[mk.FusedMoEExperts]]: if backend == Int8MoeBackend.TRITON: - from vllm.model_executor.layers.fused_moe.fused_moe import ( + from vllm.model_executor.layers.fused_moe.experts.triton_moe import ( TritonExperts, ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py index 5503d233f1..a0dde3cb7f 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py @@ -12,7 +12,7 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, ) -from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( +from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( BatchedMarlinExperts, MarlinExperts, ) @@ -42,14 +42,14 @@ def backend_to_kernel_cls( ) -> list[type[mk.FusedMoEExperts]]: """Return the experts class for the given backend, or None for NONE.""" if backend == WNA16MoEBackend.MARLIN: - from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( + from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( MarlinExperts, ) return [MarlinExperts] elif backend == WNA16MoEBackend.BATCHED_MARLIN: - from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( + from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( BatchedMarlinExperts, ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index f2742fd18b..4de0d722d5 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -124,7 +124,7 @@ def backend_to_kernel_cls( Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16, Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8, ): - from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import ( + from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( # noqa: E501 FlashInferExperts, ) @@ -160,21 +160,21 @@ def backend_to_kernel_cls( ] elif backend == Mxfp4MoeBackend.MARLIN: - from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( + from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( MarlinExperts, ) return [MarlinExperts] elif backend == Mxfp4MoeBackend.BATCHED_MARLIN: - from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( + from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( BatchedMarlinExperts, ) return [BatchedMarlinExperts] elif backend == Mxfp4MoeBackend.AITER_MXFP4_BF16: - from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( AiterExperts, ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py index 12c543b419..d22fe95f1e 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py @@ -89,7 +89,7 @@ def backend_to_kernel_cls( ] elif backend == NvFp4MoeBackend.FLASHINFER_CUTLASS: - from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import ( + from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( # noqa: E501 FlashInferExperts, ) @@ -117,7 +117,7 @@ def backend_to_kernel_cls( return [CutlassExpertsFp4] elif backend == NvFp4MoeBackend.MARLIN: - from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( + from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( MarlinExperts, ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index 8240a5e8c9..f1fd196c09 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -95,21 +95,23 @@ def backend_to_kernel_cls( return TrtLlmBf16Experts elif backend == UnquantizedMoeBackend.FLASHINFER_CUTLASS: - from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import ( + from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( # noqa: E501 FlashInferExperts, ) return FlashInferExperts elif backend == UnquantizedMoeBackend.AITER: - from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( AiterExperts, ) return AiterExperts elif backend == UnquantizedMoeBackend.TRITON: - from vllm.model_executor.layers.fused_moe.fused_moe import TritonExperts + from vllm.model_executor.layers.fused_moe.experts.triton_moe import ( + TritonExperts, + ) return TritonExperts diff --git a/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py b/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py index 74c3a62a1f..37d812a24b 100644 --- a/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py +++ b/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py @@ -14,7 +14,7 @@ from vllm.model_executor.layers.fused_moe.config import ( RoutingMethodType, get_routing_method_type, ) -from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( +from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( rocm_aiter_grouped_topk, ) from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter diff --git a/vllm/model_executor/layers/fused_moe/triton_cutlass_moe.py b/vllm/model_executor/layers/fused_moe/triton_cutlass_moe.py index 7043187893..ee3622b4eb 100644 --- a/vllm/model_executor/layers/fused_moe/triton_cutlass_moe.py +++ b/vllm/model_executor/layers/fused_moe/triton_cutlass_moe.py @@ -11,8 +11,8 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEQuantConfig, ) from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import CutlassExpertsFp8 +from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts from vllm.model_executor.layers.fused_moe.fallback import FallbackExperts -from vllm.model_executor.layers.fused_moe.fused_moe import TritonExperts from vllm.platforms import current_platform diff --git a/vllm/model_executor/layers/fused_moe/triton_deep_gemm_moe.py b/vllm/model_executor/layers/fused_moe/triton_deep_gemm_moe.py index d58dbc20e2..fbcf030a92 100644 --- a/vllm/model_executor/layers/fused_moe/triton_deep_gemm_moe.py +++ b/vllm/model_executor/layers/fused_moe/triton_deep_gemm_moe.py @@ -14,8 +14,8 @@ from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import ( _valid_deep_gemm, _valid_deep_gemm_shape, ) +from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts from vllm.model_executor.layers.fused_moe.fallback import FallbackExperts -from vllm.model_executor.layers.fused_moe.fused_moe import TritonExperts from vllm.utils.deep_gemm import ( is_deep_gemm_e8m0_used, ) diff --git a/vllm/model_executor/layers/quantization/awq_marlin.py b/vllm/model_executor/layers/quantization/awq_marlin.py index cfad1f86fa..8f455220fe 100644 --- a/vllm/model_executor/layers/quantization/awq_marlin.py +++ b/vllm/model_executor/layers/quantization/awq_marlin.py @@ -20,7 +20,7 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, ) -from vllm.model_executor.layers.fused_moe.fused_marlin_moe import fused_marlin_moe +from vllm.model_executor.layers.fused_moe.experts.marlin_moe import fused_marlin_moe from vllm.model_executor.layers.fused_moe.layer import ( FusedMoE, FusedMoEMethodBase, @@ -764,7 +764,7 @@ class AWQMarlinMoEMethod(FusedMoEMethodBase): ) from vllm.model_executor.layers.fused_moe import modular_kernel as mk - from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( + from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( BatchedMarlinExperts, MarlinExperts, ) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py index 629e1c5ef1..2ac2e28f20 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py @@ -17,7 +17,7 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( CutlassExpertsMxfp4, ) -from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( +from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( MarlinExperts, ) from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py index 8f86e687b7..10a7302dc1 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py @@ -20,7 +20,7 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEQuantConfig, int4_w4a16_moe_quant_config, ) -from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( +from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( BatchedMarlinExperts, MarlinExperts, fused_marlin_moe, diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index a14bfbc9c1..3889e376b5 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -26,7 +26,7 @@ from vllm.model_executor.layers.fused_moe.config import ( mxfp4_w4a16_moe_quant_config, ocp_mx_moe_quant_config, ) -from vllm.model_executor.layers.fused_moe.fused_marlin_moe import fused_marlin_moe +from vllm.model_executor.layers.fused_moe.experts.marlin_moe import fused_marlin_moe from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( TRITON_BACKENDS, Mxfp4MoeBackend, @@ -444,7 +444,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): shared_experts_input: torch.Tensor | None, ) -> torch.Tensor: if self.rocm_aiter_moe_enabled: - from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( rocm_aiter_fused_experts, ) @@ -909,7 +909,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod): topk_ids: torch.Tensor, shared_experts_input: torch.Tensor | None, ) -> torch.Tensor: - from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( rocm_aiter_fused_experts, ) @@ -1436,7 +1436,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): # AITER path # TODO: Refactor this to use modular MOE kernel as well. - from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( rocm_aiter_fused_experts, )