[MoE] Share apply_moe_activation support metadata (#44359)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
14e57ad47d
commit
8f158d0ee2
@@ -11,8 +11,15 @@ import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
ApplyMoEActivationConfig,
|
||||
fused_experts,
|
||||
fused_topk,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
MoEActivation,
|
||||
apply_moe_activation_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
@@ -24,8 +31,11 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
CutlassExpertsFp4,
|
||||
CutlassExpertsFp8,
|
||||
CutlassExpertsMxfp4,
|
||||
CutlassExpertsW4A8Fp8,
|
||||
run_cutlass_moe_fp8,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
@@ -53,10 +63,72 @@ MNK_FACTORS = [
|
||||
vllm_config = VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
|
||||
|
||||
def test_cutlass_moe_supports_gelu_tanh_activation_metadata():
|
||||
assert CutlassExpertsFp8._supports_activation(MoEActivation.GELU_TANH)
|
||||
assert CutlassExpertsFp4._supports_activation(MoEActivation.GELU_TANH)
|
||||
assert CutlassExpertsFp4._supports_activation(MoEActivation.GELU_TANH_NO_MUL)
|
||||
@pytest.mark.parametrize(
|
||||
"experts_cls",
|
||||
[
|
||||
CutlassExpertsFp8,
|
||||
CutlassExpertsFp4,
|
||||
CutlassExpertsMxfp4,
|
||||
CutlassExpertsW4A8Fp8,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("activation", list(MoEActivation))
|
||||
def test_cutlass_moe_activation_metadata_tracks_shared_apply(
|
||||
experts_cls: type[mk.FusedMoEExperts], activation: MoEActivation
|
||||
):
|
||||
supports_shape = experts_cls._supports_no_act_and_mul() or activation.is_gated
|
||||
expected = supports_shape and apply_moe_activation_supported(activation)
|
||||
|
||||
assert experts_cls._supports_activation(activation) == expected
|
||||
|
||||
|
||||
def test_cutlass_moe_forwards_shared_activation_parameters():
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.swiglu_limit = 7.0
|
||||
moe_config.swiglu_alpha = 1.5
|
||||
moe_config.swiglu_beta = 0.25
|
||||
moe_config.activation_situ_beta = 2.0
|
||||
moe_config.activation_situ_linear_beta = 3.0
|
||||
quant_config = FusedMoEQuantConfig.make(gemm1_alpha=1.75)
|
||||
|
||||
assert ApplyMoEActivationConfig.from_configs(
|
||||
moe_config, quant_config
|
||||
) == ApplyMoEActivationConfig(
|
||||
clamp_limit=7.0,
|
||||
alpha=1.75,
|
||||
beta=0.25,
|
||||
activation_situ_beta=2.0,
|
||||
activation_situ_linear_beta=3.0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "expected"),
|
||||
[
|
||||
("cutlass", nvfp4_oracle.NvFp4MoeBackend.VLLM_CUTLASS),
|
||||
("humming", nvfp4_oracle.NvFp4MoeBackend.HUMMING),
|
||||
],
|
||||
)
|
||||
def test_nvfp4_clamp_allows_shared_activation_backends(
|
||||
monkeypatch, backend: str, expected: nvfp4_oracle.NvFp4MoeBackend
|
||||
):
|
||||
class SupportedExperts:
|
||||
@staticmethod
|
||||
def is_supported_config(*args, **kwargs):
|
||||
return True, None
|
||||
|
||||
monkeypatch.setattr(
|
||||
nvfp4_oracle, "backend_to_kernel_cls", lambda backend: [SupportedExperts]
|
||||
)
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.moe_backend = backend
|
||||
moe_config.swiglu_limit = 7.0
|
||||
|
||||
selected, _ = nvfp4_oracle.select_nvfp4_moe_backend(
|
||||
moe_config, weight_key=None, activation_key=None
|
||||
)
|
||||
|
||||
assert selected == expected
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
|
||||
@@ -27,6 +27,11 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
MoEActivation,
|
||||
fused_topk,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
ApplyMoEActivationConfig,
|
||||
apply_moe_activation,
|
||||
apply_moe_activation_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
int4_w4a16_moe_quant_config,
|
||||
@@ -996,6 +1001,23 @@ def test_fused_marlin_moe(
|
||||
per_act_token_quant=True,
|
||||
)
|
||||
|
||||
def instance_activation(
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
*,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
output,
|
||||
input,
|
||||
activation_config=ApplyMoEActivationConfig(),
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
)
|
||||
|
||||
marlin_output = fused_marlin_moe(
|
||||
a,
|
||||
w1_data.qweight,
|
||||
@@ -1021,6 +1043,7 @@ def test_fused_marlin_moe(
|
||||
input_dtype=a_dtype,
|
||||
quant_type_id=b_type.id,
|
||||
is_k_full=is_k_full,
|
||||
activation_func=instance_activation,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(marlin_output, torch_output, atol=4e-2, rtol=0)
|
||||
@@ -1249,6 +1272,51 @@ def _make_humming_indexed_experts(activation: MoEActivation):
|
||||
return experts
|
||||
|
||||
|
||||
@pytest.mark.parametrize("activation", list(MoEActivation))
|
||||
def test_humming_activation_metadata_tracks_shared_apply(activation: MoEActivation):
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingExpertsBase,
|
||||
)
|
||||
|
||||
assert HummingExpertsBase._supports_activation(
|
||||
activation
|
||||
) == apply_moe_activation_supported(activation)
|
||||
|
||||
|
||||
def test_humming_delegates_to_instance_activation():
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingExpertsBase,
|
||||
)
|
||||
|
||||
activation_func = Mock()
|
||||
activation_config = ApplyMoEActivationConfig(
|
||||
clamp_limit=7.0,
|
||||
alpha=1.5,
|
||||
beta=0.25,
|
||||
activation_situ_beta=2.0,
|
||||
activation_situ_linear_beta=3.0,
|
||||
)
|
||||
experts = SimpleNamespace(
|
||||
activation=activation_func,
|
||||
activation_config=activation_config,
|
||||
)
|
||||
input = torch.empty(1, 2)
|
||||
output = torch.empty(1, 1)
|
||||
|
||||
HummingExpertsBase.apply_activation(
|
||||
experts, MoEActivation.SWIGLUOAI_UNINTERLEAVE, output, input
|
||||
)
|
||||
|
||||
activation_func.assert_called_once_with(
|
||||
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
input=input,
|
||||
output=output,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
|
||||
@@ -11,7 +11,12 @@ import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
ApplyMoEActivationConfig,
|
||||
MoEActivation,
|
||||
apply_moe_activation,
|
||||
apply_moe_activation_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
)
|
||||
@@ -31,6 +36,49 @@ NO_MUL_ACTIVATIONS = [
|
||||
]
|
||||
|
||||
|
||||
def test_apply_moe_activation_supported_contract():
|
||||
supported = {
|
||||
activation
|
||||
for activation in MoEActivation
|
||||
if apply_moe_activation_supported(activation)
|
||||
}
|
||||
|
||||
assert supported == set(MoEActivation) - {MoEActivation.RELU2}
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
activation
|
||||
for activation in MoEActivation
|
||||
if apply_moe_activation_supported(activation)
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_supported_apply_moe_activation_executes(activation: MoEActivation):
|
||||
output_width = 64
|
||||
input_width = output_width * 2 if activation.is_gated else output_width
|
||||
input = torch.randn(2, input_width, device="cuda", dtype=torch.bfloat16)
|
||||
output = torch.empty(2, output_width, device="cuda", dtype=torch.bfloat16)
|
||||
activation_config = ApplyMoEActivationConfig()
|
||||
if activation == MoEActivation.SITU:
|
||||
activation_config = ApplyMoEActivationConfig(activation_situ_beta=1.0)
|
||||
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
activation_config = ApplyMoEActivationConfig(clamp_limit=7.0)
|
||||
|
||||
apply_moe_activation(activation, output, input, activation_config=activation_config)
|
||||
|
||||
assert torch.isfinite(output).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("activation", list(MoEActivation))
|
||||
def test_triton_activation_metadata_tracks_shared_apply(activation: MoEActivation):
|
||||
assert TritonExperts._supports_activation(
|
||||
activation
|
||||
) == apply_moe_activation_supported(activation)
|
||||
|
||||
|
||||
def make_test_tensors(
|
||||
m: int,
|
||||
n: int,
|
||||
|
||||
Reference in New Issue
Block a user