[MoE] Share apply_moe_activation support metadata (#44359)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
14e57ad47d
commit
8f158d0ee2
@@ -2,11 +2,19 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MoE activation function enum and utilities."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
|
||||
|
||||
class MoEActivation(Enum):
|
||||
"""Activation functions for MoE layers."""
|
||||
@@ -109,6 +117,66 @@ def activation_without_mul(activation: str) -> str:
|
||||
return MoEActivation.from_str(activation).without_mul().value
|
||||
|
||||
|
||||
_APPLY_MOE_ACTIVATIONS = frozenset(
|
||||
{
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SITU,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
MoEActivation.SILU_NO_MUL,
|
||||
MoEActivation.GELU_NO_MUL,
|
||||
MoEActivation.GELU_TANH_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def apply_moe_activation_supported(activation: MoEActivation) -> bool:
|
||||
"""Whether ``apply_moe_activation`` supports an activation."""
|
||||
return activation in _APPLY_MOE_ACTIVATIONS
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApplyMoEActivationConfig:
|
||||
"""Configuration forwarded to ``apply_moe_activation``."""
|
||||
|
||||
clamp_limit: float | None = None
|
||||
alpha: float = 1.0
|
||||
beta: float = 0.0
|
||||
activation_situ_beta: float | None = None
|
||||
activation_situ_linear_beta: float | None = None
|
||||
|
||||
@classmethod
|
||||
def from_configs(
|
||||
cls,
|
||||
moe_config: "FusedMoEConfig",
|
||||
quant_config: "FusedMoEQuantConfig",
|
||||
) -> "ApplyMoEActivationConfig":
|
||||
"""Build from the model and quantization configurations."""
|
||||
clamp_limit = quant_config.gemm1_clamp_limit
|
||||
if clamp_limit is None:
|
||||
clamp_limit = moe_config.swiglu_limit
|
||||
alpha = quant_config.gemm1_alpha
|
||||
if alpha is None:
|
||||
alpha = moe_config.swiglu_alpha
|
||||
beta = quant_config.gemm1_beta
|
||||
if beta is None:
|
||||
beta = moe_config.swiglu_beta
|
||||
return cls(
|
||||
clamp_limit=clamp_limit,
|
||||
alpha=1.0 if alpha is None else alpha,
|
||||
beta=0.0 if beta is None else beta,
|
||||
activation_situ_beta=moe_config.activation_situ_beta,
|
||||
activation_situ_linear_beta=moe_config.activation_situ_linear_beta,
|
||||
)
|
||||
|
||||
|
||||
_DEFAULT_APPLY_MOE_ACTIVATION_CONFIG = ApplyMoEActivationConfig()
|
||||
|
||||
|
||||
def silu_and_mul_with_clamp(
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
@@ -130,20 +198,21 @@ def apply_moe_activation(
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
*,
|
||||
clamp_limit: float | None = None,
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
activation_config: ApplyMoEActivationConfig | None = None,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply MoE activation function.
|
||||
|
||||
``clamp_limit``/``alpha``/``beta`` (from the quant config) drive the clamped
|
||||
SwiGLU kernels: ``SILU`` + ``clamp_limit`` and ``SWIGLUOAI_UNINTERLEAVE`` both
|
||||
map to ``silu_and_mul_with_clamp``. Other activations ignore them.
|
||||
The configuration drives specialized activation behavior. Routing tensors
|
||||
remain per-call inputs because they depend on the current token assignment.
|
||||
"""
|
||||
config = (
|
||||
_DEFAULT_APPLY_MOE_ACTIVATION_CONFIG
|
||||
if activation_config is None
|
||||
else activation_config
|
||||
)
|
||||
|
||||
assert input.dim() == 2, "Input must be 2D"
|
||||
assert output.dim() == 2, "Output must be 2D"
|
||||
if activation.is_gated:
|
||||
@@ -159,8 +228,10 @@ def apply_moe_activation(
|
||||
|
||||
# Activations with gated multiplication (gate × activation(up))
|
||||
if activation == MoEActivation.SILU:
|
||||
if clamp_limit is not None:
|
||||
silu_and_mul_with_clamp(output, input, clamp_limit, topk_ids, expert_map)
|
||||
if config.clamp_limit is not None:
|
||||
silu_and_mul_with_clamp(
|
||||
output, input, config.clamp_limit, topk_ids, expert_map
|
||||
)
|
||||
else:
|
||||
torch.ops._C.silu_and_mul(output, input)
|
||||
elif activation == MoEActivation.GELU:
|
||||
@@ -175,23 +246,27 @@ def apply_moe_activation(
|
||||
# bypassed the config plumbing, so fail rather than silently use 1.0.
|
||||
# linear_beta is genuinely optional: <= 0 signals "unset" to the kernel
|
||||
# (up passed through), matching SituAndMul(linear_beta=None).
|
||||
assert activation_situ_beta is not None, (
|
||||
assert config.activation_situ_beta is not None, (
|
||||
"SITU requires activation_situ_beta from FusedMoEConfig"
|
||||
)
|
||||
torch.ops._C.situ_and_mul(
|
||||
output,
|
||||
input,
|
||||
activation_situ_beta,
|
||||
config.activation_situ_beta,
|
||||
-1.0
|
||||
if activation_situ_linear_beta is None
|
||||
else activation_situ_linear_beta,
|
||||
if config.activation_situ_linear_beta is None
|
||||
else config.activation_situ_linear_beta,
|
||||
)
|
||||
elif activation == MoEActivation.SWIGLUOAI:
|
||||
torch.ops._C.swigluoai_and_mul(output, input)
|
||||
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
# SwiGLU-OAI on packed w13 (gate = first half, up = second half).
|
||||
assert clamp_limit is not None, "SWIGLUOAI_UNINTERLEAVE requires clamp_limit"
|
||||
torch.ops._C.silu_and_mul_with_clamp(output, input, clamp_limit, alpha, beta)
|
||||
assert config.clamp_limit is not None, (
|
||||
"SWIGLUOAI_UNINTERLEAVE requires clamp_limit"
|
||||
)
|
||||
torch.ops._C.silu_and_mul_with_clamp(
|
||||
output, input, config.clamp_limit, config.alpha, config.beta
|
||||
)
|
||||
elif activation == MoEActivation.SWIGLUSTEP:
|
||||
from vllm.model_executor.layers.activation import swiglustep_and_mul_triton
|
||||
|
||||
|
||||
Reference in New Issue
Block a user