[Performance] Add Triton kernel for Gemma3n sparse GELU (#48498)

Signed-off-by: BWAAEEEK <jooho414@gmail.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
JooHo Lee
2026-09-14 07:42:15 +00:00
committed by GitHub
co-authored by Andreas Karatzas Cyrus Leung
parent e6b4e47d2d
commit 78e84261ab
15 changed files with 555 additions and 21 deletions
+3 -15
View File
@@ -8,6 +8,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm import ir
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.platforms import CpuArchEnum, current_platform
@@ -332,24 +333,11 @@ class GeluAndMulSparse(CustomOp):
raise ValueError("activation_sparsity is 0.0. Please use GeluAndMul.")
target_sparsity_tensor = torch.tensor(activation_sparsity, dtype=torch.float32)
normal_dist = torch.distributions.normal.Normal(0, 1)
self.std_multiplier = normal_dist.icdf(target_sparsity_tensor)
def _gaussian_topk(self, x: torch.Tensor) -> torch.Tensor:
"""Get % sparse percentile of the Gaussian distribution."""
# NOTE(rob): for TP>1, we could all-gather to get the means/std.
# But we do not do this because in expectation they are the same
# and in practice the eval scores are good without gathering.
mean = torch.mean(x, dim=-1, keepdim=True)
std = torch.std(x, dim=-1, keepdim=True, unbiased=False)
cutoff_x = mean + std * self.std_multiplier
return nn.functional.relu(x - cutoff_x)
self.std_multiplier = normal_dist.icdf(target_sparsity_tensor).item()
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
"""PyTorch-native implementation equivalent to forward()."""
d = x.shape[-1] // 2
out = self._gaussian_topk(x[..., :d])
out = F.gelu(out, approximate=self.approximate)
return out * x[..., d:]
return ir.ops.gelu_and_mul_sparse(x, self.std_multiplier, self.approximate)
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
return self.forward_native(x)