[Refactor] Replace activation: str with MoEActivation enum (#33843)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
@@ -4,7 +4,6 @@ import functools
|
||||
from math import prod
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
@@ -341,65 +340,6 @@ def _validate_scale_shape(
|
||||
assert a_scale.shape == expected, f"{a_scale.shape} == {expected}"
|
||||
|
||||
|
||||
def activation_without_mul(activation: str) -> str:
|
||||
return activation + "_no_mul"
|
||||
|
||||
|
||||
RELU2_NO_MUL: str = activation_without_mul("relu2")
|
||||
SILU_NO_MUL: str = activation_without_mul("silu")
|
||||
GELU_NO_MUL: str = activation_without_mul("gelu")
|
||||
|
||||
|
||||
def apply_moe_activation(
|
||||
activation: str,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply MoE activation function.
|
||||
|
||||
For *_and_mul activations (silu, gelu, swigluoai):
|
||||
- Expects output.size(-1) * 2 == input.size(-1)
|
||||
|
||||
For *_no_mul activations (silu_no_mul, gelu_no_mul, relu2_no_mul):
|
||||
- Expects output.size(-1) == input.size(-1)
|
||||
"""
|
||||
is_no_mul = activation.endswith("_no_mul")
|
||||
if is_no_mul:
|
||||
assert output.size(-1) == input.size(-1), (
|
||||
f"{activation} expects equal sizes: {output.size(-1)} vs {input.size(-1)}"
|
||||
)
|
||||
else:
|
||||
assert output.size(-1) * 2 == input.size(-1), (
|
||||
f"{activation} expects 2x ratio: {output.size(-1) * 2} vs {input.size(-1)}"
|
||||
)
|
||||
|
||||
# Activations with gated multiplication (gate × activation(up))
|
||||
if activation == "silu":
|
||||
torch.ops._C.silu_and_mul(output, input)
|
||||
elif activation == "gelu":
|
||||
torch.ops._C.gelu_and_mul(output, input)
|
||||
elif activation == "swigluoai":
|
||||
torch.ops._C.swigluoai_and_mul(output, input)
|
||||
elif activation == "swiglustep":
|
||||
from vllm.model_executor.layers.activation import swiglustep_and_mul_triton
|
||||
|
||||
swiglustep_and_mul_triton(output, input)
|
||||
|
||||
# Activations without gated multiplication
|
||||
elif activation == SILU_NO_MUL:
|
||||
output.copy_(F.silu(input))
|
||||
elif activation == GELU_NO_MUL:
|
||||
output.copy_(F.gelu(input))
|
||||
elif activation == RELU2_NO_MUL:
|
||||
F.relu(input, inplace=True)
|
||||
torch.square(input, out=output)
|
||||
else:
|
||||
raise ValueError(f"Unsupported FusedMoe activation: {activation}")
|
||||
|
||||
return output
|
||||
|
||||
|
||||
# Torch custom ops can't deal with outputs aliasing inputs so we need to
|
||||
# disable inplace for torch >= 2.9.
|
||||
# See https://github.com/vllm-project/vllm/issues/26378
|
||||
|
||||
Reference in New Issue
Block a user