[FIX] Add NO_MUL activation support for modular kernel path (#31528)
Signed-off-by: dafrimi <dafrimi@nvidia.com> Signed-off-by: <> Co-authored-by: root <root@gpu-267.slurm-workers-slurm.slurm.svc.cluster.local> Co-authored-by: root <root@gpu-537.slurm-workers-slurm.slurm.svc.cluster.local> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: root <root@pool0-01777.cm.cluster>
This commit is contained in:
co-authored by
root
root
Michael Goin
root
parent
6bc9c8473e
commit
3f72639d36
@@ -4,6 +4,7 @@ import functools
|
||||
from math import prod
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
@@ -324,6 +325,55 @@ def activation_without_mul(activation: str) -> str:
|
||||
return activation + "_no_mul"
|
||||
|
||||
|
||||
RELU2_NO_MUL: str = activation_without_mul("relu2")
|
||||
SILU_NO_MUL: str = activation_without_mul("silu")
|
||||
GELU_NO_MUL: str = activation_without_mul("gelu")
|
||||
|
||||
|
||||
def apply_moe_activation(
|
||||
activation: str,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply MoE activation function.
|
||||
|
||||
For *_and_mul activations (silu, gelu, swigluoai):
|
||||
- Expects output.size(-1) * 2 == input.size(-1)
|
||||
|
||||
For *_no_mul activations (silu_no_mul, gelu_no_mul, relu2_no_mul):
|
||||
- Expects output.size(-1) == input.size(-1)
|
||||
"""
|
||||
is_no_mul = activation.endswith("_no_mul")
|
||||
if is_no_mul:
|
||||
assert output.size(-1) == input.size(-1), (
|
||||
f"{activation} expects equal sizes: {output.size(-1)} vs {input.size(-1)}"
|
||||
)
|
||||
else:
|
||||
assert output.size(-1) * 2 == input.size(-1), (
|
||||
f"{activation} expects 2x ratio: {output.size(-1) * 2} vs {input.size(-1)}"
|
||||
)
|
||||
|
||||
# Activations with gated multiplication (gate × activation(up))
|
||||
if activation == "silu":
|
||||
torch.ops._C.silu_and_mul(output, input)
|
||||
elif activation == "gelu":
|
||||
torch.ops._C.gelu_and_mul(output, input)
|
||||
elif activation == "swigluoai":
|
||||
torch.ops._C.swigluoai_and_mul(output, input)
|
||||
# Activations without gated multiplication
|
||||
elif activation == SILU_NO_MUL:
|
||||
output.copy_(F.silu(input))
|
||||
elif activation == GELU_NO_MUL:
|
||||
output.copy_(F.gelu(input))
|
||||
elif activation == RELU2_NO_MUL:
|
||||
torch.square(F.relu(input), out=output)
|
||||
else:
|
||||
raise ValueError(f"Unsupported FusedMoe activation: {activation}")
|
||||
|
||||
return output
|
||||
|
||||
|
||||
# Torch custom ops can't deal with outputs aliasing inputs so we need to
|
||||
# disable inplace for torch >= 2.9.
|
||||
# See https://github.com/vllm-project/vllm/issues/26378
|
||||
|
||||
Reference in New Issue
Block a user