[MoE] Share apply_moe_activation support metadata (#44359)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Michael Goin
2026-08-05 12:05:47 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 14e57ad47d
commit 8f158d0ee2
13 changed files with 425 additions and 241 deletions
@@ -2,11 +2,19 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MoE activation function enum and utilities."""
from dataclasses import dataclass
from enum import Enum
from typing import TYPE_CHECKING
import torch
import torch.nn.functional as F
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
class MoEActivation(Enum):
"""Activation functions for MoE layers."""
@@ -109,6 +117,66 @@ def activation_without_mul(activation: str) -> str:
return MoEActivation.from_str(activation).without_mul().value
_APPLY_MOE_ACTIVATIONS = frozenset(
{
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SITU,
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
MoEActivation.SWIGLUSTEP,
MoEActivation.SILU_NO_MUL,
MoEActivation.GELU_NO_MUL,
MoEActivation.GELU_TANH_NO_MUL,
MoEActivation.RELU2_NO_MUL,
}
)
def apply_moe_activation_supported(activation: MoEActivation) -> bool:
"""Whether ``apply_moe_activation`` supports an activation."""
return activation in _APPLY_MOE_ACTIVATIONS
@dataclass(frozen=True)
class ApplyMoEActivationConfig:
"""Configuration forwarded to ``apply_moe_activation``."""
clamp_limit: float | None = None
alpha: float = 1.0
beta: float = 0.0
activation_situ_beta: float | None = None
activation_situ_linear_beta: float | None = None
@classmethod
def from_configs(
cls,
moe_config: "FusedMoEConfig",
quant_config: "FusedMoEQuantConfig",
) -> "ApplyMoEActivationConfig":
"""Build from the model and quantization configurations."""
clamp_limit = quant_config.gemm1_clamp_limit
if clamp_limit is None:
clamp_limit = moe_config.swiglu_limit
alpha = quant_config.gemm1_alpha
if alpha is None:
alpha = moe_config.swiglu_alpha
beta = quant_config.gemm1_beta
if beta is None:
beta = moe_config.swiglu_beta
return cls(
clamp_limit=clamp_limit,
alpha=1.0 if alpha is None else alpha,
beta=0.0 if beta is None else beta,
activation_situ_beta=moe_config.activation_situ_beta,
activation_situ_linear_beta=moe_config.activation_situ_linear_beta,
)
_DEFAULT_APPLY_MOE_ACTIVATION_CONFIG = ApplyMoEActivationConfig()
def silu_and_mul_with_clamp(
output: torch.Tensor,
input: torch.Tensor,
@@ -130,20 +198,21 @@ def apply_moe_activation(
output: torch.Tensor,
input: torch.Tensor,
*,
clamp_limit: float | None = None,
alpha: float = 1.0,
beta: float = 0.0,
activation_config: ApplyMoEActivationConfig | None = None,
topk_ids: torch.Tensor | None = None,
expert_map: torch.Tensor | None = None,
activation_situ_beta: float | None = None,
activation_situ_linear_beta: float | None = None,
) -> torch.Tensor:
"""Apply MoE activation function.
``clamp_limit``/``alpha``/``beta`` (from the quant config) drive the clamped
SwiGLU kernels: ``SILU`` + ``clamp_limit`` and ``SWIGLUOAI_UNINTERLEAVE`` both
map to ``silu_and_mul_with_clamp``. Other activations ignore them.
The configuration drives specialized activation behavior. Routing tensors
remain per-call inputs because they depend on the current token assignment.
"""
config = (
_DEFAULT_APPLY_MOE_ACTIVATION_CONFIG
if activation_config is None
else activation_config
)
assert input.dim() == 2, "Input must be 2D"
assert output.dim() == 2, "Output must be 2D"
if activation.is_gated:
@@ -159,8 +228,10 @@ def apply_moe_activation(
# Activations with gated multiplication (gate × activation(up))
if activation == MoEActivation.SILU:
if clamp_limit is not None:
silu_and_mul_with_clamp(output, input, clamp_limit, topk_ids, expert_map)
if config.clamp_limit is not None:
silu_and_mul_with_clamp(
output, input, config.clamp_limit, topk_ids, expert_map
)
else:
torch.ops._C.silu_and_mul(output, input)
elif activation == MoEActivation.GELU:
@@ -175,23 +246,27 @@ def apply_moe_activation(
# bypassed the config plumbing, so fail rather than silently use 1.0.
# linear_beta is genuinely optional: <= 0 signals "unset" to the kernel
# (up passed through), matching SituAndMul(linear_beta=None).
assert activation_situ_beta is not None, (
assert config.activation_situ_beta is not None, (
"SITU requires activation_situ_beta from FusedMoEConfig"
)
torch.ops._C.situ_and_mul(
output,
input,
activation_situ_beta,
config.activation_situ_beta,
-1.0
if activation_situ_linear_beta is None
else activation_situ_linear_beta,
if config.activation_situ_linear_beta is None
else config.activation_situ_linear_beta,
)
elif activation == MoEActivation.SWIGLUOAI:
torch.ops._C.swigluoai_and_mul(output, input)
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
# SwiGLU-OAI on packed w13 (gate = first half, up = second half).
assert clamp_limit is not None, "SWIGLUOAI_UNINTERLEAVE requires clamp_limit"
torch.ops._C.silu_and_mul_with_clamp(output, input, clamp_limit, alpha, beta)
assert config.clamp_limit is not None, (
"SWIGLUOAI_UNINTERLEAVE requires clamp_limit"
)
torch.ops._C.silu_and_mul_with_clamp(
output, input, config.clamp_limit, config.alpha, config.beta
)
elif activation == MoEActivation.SWIGLUSTEP:
from vllm.model_executor.layers.activation import swiglustep_and_mul_triton