[Kernel][MoE] Add GELU_TANH to CPU, CUTLASS, and WNA16 MoE backends (#42027)
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com> Co-authored-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
@@ -20,7 +20,12 @@ EXPERT_NUM = [
|
||||
HIDDEN_DIM = [128, 2880]
|
||||
INTERMEDIATE_DIM = [128, 2880]
|
||||
BATCH_SIZE = [1, 64, 256]
|
||||
ACT = [MoEActivation.SILU, MoEActivation.SWIGLUOAI, MoEActivation.GELU]
|
||||
ACT = [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
]
|
||||
USE_BIAS = [True, False]
|
||||
ISA = ["amx", "vec"] if torch.cpu._is_amx_tile_supported() else ["vec"]
|
||||
DTYPE = [torch.bfloat16]
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
fp8_w8a8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
CutlassExpertsFp4,
|
||||
CutlassExpertsFp8,
|
||||
run_cutlass_moe_fp8,
|
||||
)
|
||||
@@ -52,6 +53,12 @@ MNK_FACTORS = [
|
||||
vllm_config = VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
|
||||
|
||||
def test_cutlass_moe_supports_gelu_tanh_activation_metadata():
|
||||
assert CutlassExpertsFp8._supports_activation(MoEActivation.GELU_TANH)
|
||||
assert CutlassExpertsFp4._supports_activation(MoEActivation.GELU_TANH)
|
||||
assert CutlassExpertsFp4._supports_activation(MoEActivation.GELU_TANH_NO_MUL)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class MOETensors:
|
||||
a: torch.Tensor
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Method
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test on CUDA")
|
||||
def test_moe_wna16_apply_passes_layer_activation(monkeypatch):
|
||||
captured_kwargs = {}
|
||||
|
||||
def fake_fused_experts(*args, **kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
return torch.empty(1, 2)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.fused_moe.fused_experts",
|
||||
fake_fused_experts,
|
||||
)
|
||||
|
||||
method = object.__new__(MoeWNA16Method)
|
||||
method.moe = SimpleNamespace(disable_inplace=False)
|
||||
method.moe_quant_config = object()
|
||||
layer = SimpleNamespace(
|
||||
w13_qweight=torch.empty(1, 2),
|
||||
w2_qweight=torch.empty(1, 2),
|
||||
activation=MoEActivation.GELU_TANH,
|
||||
apply_router_weight_on_input=False,
|
||||
global_num_experts=1,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
output = method.apply(
|
||||
layer,
|
||||
x=torch.empty(1, 2),
|
||||
topk_weights=torch.empty(1, 1),
|
||||
topk_ids=torch.empty(1, 1, dtype=torch.int32),
|
||||
shared_experts=None,
|
||||
shared_experts_input=None,
|
||||
)
|
||||
|
||||
assert output.shape == (1, 2)
|
||||
assert captured_kwargs["activation"] is MoEActivation.GELU_TANH
|
||||
Reference in New Issue
Block a user