[Performance] Add Triton kernel for Gemma3n sparse GELU (#48498)

Signed-off-by: BWAAEEEK <jooho414@gmail.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
JooHo Lee
2026-09-14 07:42:15 +00:00
committed by GitHub
co-authored by Andreas Karatzas Cyrus Leung
parent e6b4e47d2d
commit 78e84261ab
15 changed files with 555 additions and 21 deletions
+3
View File
@@ -34,6 +34,9 @@ class IrOpPriorityConfig:
fused_add_rms_norm: list[str] = Field(default_factory=list)
"""Priority list for vllm.ir.ops.fused_add_rms_norm"""
gelu_and_mul_sparse: list[str] = Field(default_factory=list)
"""Priority list for vllm.ir.ops.gelu_and_mul_sparse"""
def compute_hash(self) -> str:
"""
Produces a hash unique to the pass configuration.
+2 -1
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .activation import gelu_and_mul_sparse
from .layernorm import fused_add_rms_norm, rms_norm
__all__ = ["rms_norm", "fused_add_rms_norm"]
__all__ = ["rms_norm", "fused_add_rms_norm", "gelu_and_mul_sparse"]
+34
View File
@@ -0,0 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn.functional as F
from torch import Tensor
from ..op import register_op
@register_op
def gelu_and_mul_sparse(
x: Tensor, std_multiplier: float, approximate: str = "none"
) -> Tensor:
"""Apply Gaussian sparsification, GELU, and gated multiplication."""
d = x.shape[-1] // 2
gate = x[..., :d]
# Statistics intentionally remain local to each tensor-parallel shard.
mean = torch.mean(gate, dim=-1, keepdim=True)
std = torch.std(gate, dim=-1, keepdim=True, unbiased=False)
sparse_gate = F.relu(gate - (mean + std * std_multiplier))
return F.gelu(sparse_gate, approximate=approximate) * x[..., d:]
@gelu_and_mul_sparse.register_input_generator
def _gelu_and_mul_sparse_input_generator(
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
std_multiplier: float = 1.6448536269514722,
approximate: str = "tanh",
) -> tuple:
x = torch.randn(num_tokens, hidden_size * 2, dtype=dtype)
return x, std_multiplier, approximate
+2 -2
View File
@@ -2,6 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Kernel implementations for vLLM."""
from . import aiter_ops, oink_ops, vllm_c
from . import aiter_ops, oink_ops, triton, vllm_c
__all__ = ["vllm_c", "aiter_ops", "oink_ops"]
__all__ = ["vllm_c", "aiter_ops", "oink_ops", "triton"]
+4
View File
@@ -1,3 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton kernel implementations."""
from . import activation
__all__ = ["activation"]
+110
View File
@@ -0,0 +1,110 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch import Tensor
from torch.library import triton_op, wrap_triton
from vllm import ir
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON, tl, triton
_MAX_INTERMEDIATE_SIZE = 32768
_SUPPORTED_DTYPES = {torch.float16, torch.bfloat16, torch.float32}
@triton.jit
def _gelu_and_mul_sparse_kernel(
x_ptr,
output_ptr,
d: tl.constexpr,
std_multiplier,
BLOCK_SIZE: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
offsets = tl.arange(0, BLOCK_SIZE)
mask = offsets < d
row_start = row * 2 * d
gate = tl.load(x_ptr + row_start + offsets, mask=mask, other=0.0).to(tl.float32)
# TODO: Investigate delaying this load to reduce spills for large reductions
# without regressing smaller shapes.
up = tl.load(x_ptr + row_start + d + offsets, mask=mask, other=0.0).to(tl.float32)
input_dtype = x_ptr.dtype.element_ty
mean = tl.sum(gate, axis=0) / d
centered = tl.where(mask, gate - mean, 0.0)
variance = tl.sum(centered * centered, axis=0) / d
# Match the input-dtype rounding at each native PyTorch op boundary.
mean = mean.to(input_dtype).to(tl.float32)
std = tl.sqrt(variance).to(input_dtype).to(tl.float32)
scaled_std = (std * std_multiplier).to(input_dtype).to(tl.float32)
cutoff = (mean + scaled_std).to(input_dtype).to(tl.float32)
sparse_gate = (gate - cutoff).to(input_dtype).to(tl.float32)
sparse_gate = tl.where(sparse_gate < 0.0, 0.0, sparse_gate)
inner = 0.7978845608028654 * (
sparse_gate + 0.044715 * sparse_gate * sparse_gate * sparse_gate
)
tanh = 2.0 * tl.sigmoid(2.0 * inner) - 1.0
activated = 0.5 * sparse_gate * (1.0 + tanh)
activated = activated.to(input_dtype).to(tl.float32)
tl.store(output_ptr + row * d + offsets, activated * up, mask=mask)
def _supports_gelu_and_mul_sparse(
x: Tensor, std_multiplier: float, approximate: str = "none"
) -> bool:
del std_multiplier
if (
x.device.type != "cuda"
or x.dtype not in _SUPPORTED_DTYPES
or not x.is_contiguous()
or x.ndim == 0
or approximate != "tanh"
):
return False
last_dim = x.shape[-1]
return (
last_dim > 0 and last_dim % 2 == 0 and last_dim // 2 <= _MAX_INTERMEDIATE_SIZE
)
@triton_op(
"vllm::gelu_and_mul_sparse_triton",
mutates_args=(),
)
def _gelu_and_mul_sparse_triton_op(x: Tensor, std_multiplier: float) -> Tensor:
d = x.shape[-1] // 2
output = torch.empty((*x.shape[:-1], d), dtype=x.dtype, device=x.device)
if output.numel() == 0:
return output
n_rows = x.numel() // x.shape[-1]
def grid(meta):
return (n_rows,)
block_size = triton.next_power_of_2(d)
num_warps = min(max(block_size // 256, 1), 8)
wrap_triton(_gelu_and_mul_sparse_kernel)[grid](
x,
output,
d=d,
std_multiplier=std_multiplier,
BLOCK_SIZE=block_size,
num_warps=num_warps,
)
return output
@ir.ops.gelu_and_mul_sparse.register_impl(
"triton",
supported=HAS_TRITON and current_platform.is_cuda(),
supports_args=_supports_gelu_and_mul_sparse,
)
def gelu_and_mul_sparse_triton(
x: Tensor, std_multiplier: float, approximate: str = "none"
) -> Tensor:
assert approximate == "tanh"
return _gelu_and_mul_sparse_triton_op(x, std_multiplier)
+3 -15
View File
@@ -8,6 +8,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm import ir
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.platforms import CpuArchEnum, current_platform
@@ -332,24 +333,11 @@ class GeluAndMulSparse(CustomOp):
raise ValueError("activation_sparsity is 0.0. Please use GeluAndMul.")
target_sparsity_tensor = torch.tensor(activation_sparsity, dtype=torch.float32)
normal_dist = torch.distributions.normal.Normal(0, 1)
self.std_multiplier = normal_dist.icdf(target_sparsity_tensor)
def _gaussian_topk(self, x: torch.Tensor) -> torch.Tensor:
"""Get % sparse percentile of the Gaussian distribution."""
# NOTE(rob): for TP>1, we could all-gather to get the means/std.
# But we do not do this because in expectation they are the same
# and in practice the eval scores are good without gathering.
mean = torch.mean(x, dim=-1, keepdim=True)
std = torch.std(x, dim=-1, keepdim=True, unbiased=False)
cutoff_x = mean + std * self.std_multiplier
return nn.functional.relu(x - cutoff_x)
self.std_multiplier = normal_dist.icdf(target_sparsity_tensor).item()
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
"""PyTorch-native implementation equivalent to forward()."""
d = x.shape[-1] // 2
out = self._gaussian_topk(x[..., :d])
out = F.gelu(out, approximate=self.approximate)
return out * x[..., d:]
return ir.ops.gelu_and_mul_sparse(x, self.std_multiplier, self.approximate)
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
return self.forward_native(x)
+4 -1
View File
@@ -755,7 +755,10 @@ class CudaPlatformBase(Platform):
rms_norm = ["oink"] + default
return IrOpPriorityConfig.with_default(
default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm
default,
rms_norm=rms_norm,
fused_add_rms_norm=rms_norm,
gelu_and_mul_sparse=["triton", "native"],
)
@classmethod
+4 -1
View File
@@ -1161,7 +1161,10 @@ class RocmPlatform(Platform):
rms_norm = default
return IrOpPriorityConfig.with_default(
default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm
default,
rms_norm=rms_norm,
fused_add_rms_norm=rms_norm,
gelu_and_mul_sparse=["native"],
)
@classmethod
+1 -1
View File
@@ -491,7 +491,7 @@ class XPUPlatform(Platform):
using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE
default = ["native"] if using_inductor else ["vllm_c", "native"]
return IrOpPriorityConfig.with_default(default)
return IrOpPriorityConfig.with_default(default, gelu_and_mul_sparse=["native"])
@classmethod
def device_count(cls) -> int: