[Kernel][MoE] Add GELU_TANH to CPU, CUTLASS, and WNA16 MoE backends (#42027)

Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
Co-authored-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
SeongJun Lee
2026-06-02 17:12:08 -04:00
committed by GitHub
co-authored by lesj0610
parent e15f20258b
commit 3099de3617
7 changed files with 119 additions and 7 deletions
+6 -1
View File
@@ -20,7 +20,12 @@ EXPERT_NUM = [
HIDDEN_DIM = [128, 2880]
INTERMEDIATE_DIM = [128, 2880]
BATCH_SIZE = [1, 64, 256]
ACT = [MoEActivation.SILU, MoEActivation.SWIGLUOAI, MoEActivation.GELU]
ACT = [
MoEActivation.SILU,
MoEActivation.SWIGLUOAI,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
]
USE_BIAS = [True, False]
ISA = ["amx", "vec"] if torch.cpu._is_amx_tile_supported() else ["vec"]
DTYPE = [torch.bfloat16]
+7
View File
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.fused_moe.config import (
fp8_w8a8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsFp4,
CutlassExpertsFp8,
run_cutlass_moe_fp8,
)
@@ -52,6 +53,12 @@ MNK_FACTORS = [
vllm_config = VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
def test_cutlass_moe_supports_gelu_tanh_activation_metadata():
assert CutlassExpertsFp8._supports_activation(MoEActivation.GELU_TANH)
assert CutlassExpertsFp4._supports_activation(MoEActivation.GELU_TANH)
assert CutlassExpertsFp4._supports_activation(MoEActivation.GELU_TANH_NO_MUL)
@dataclasses.dataclass
class MOETensors:
a: torch.Tensor