[Performance] Add Triton kernel for Gemma3n sparse GELU (#48498)
Signed-off-by: BWAAEEEK <jooho414@gmail.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Andreas Karatzas
Cyrus Leung
parent
e6b4e47d2d
commit
78e84261ab
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from .activation import gelu_and_mul_sparse
|
||||
from .layernorm import fused_add_rms_norm, rms_norm
|
||||
|
||||
__all__ = ["rms_norm", "fused_add_rms_norm"]
|
||||
__all__ = ["rms_norm", "fused_add_rms_norm", "gelu_and_mul_sparse"]
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import Tensor
|
||||
|
||||
from ..op import register_op
|
||||
|
||||
|
||||
@register_op
|
||||
def gelu_and_mul_sparse(
|
||||
x: Tensor, std_multiplier: float, approximate: str = "none"
|
||||
) -> Tensor:
|
||||
"""Apply Gaussian sparsification, GELU, and gated multiplication."""
|
||||
d = x.shape[-1] // 2
|
||||
gate = x[..., :d]
|
||||
# Statistics intentionally remain local to each tensor-parallel shard.
|
||||
mean = torch.mean(gate, dim=-1, keepdim=True)
|
||||
std = torch.std(gate, dim=-1, keepdim=True, unbiased=False)
|
||||
sparse_gate = F.relu(gate - (mean + std * std_multiplier))
|
||||
return F.gelu(sparse_gate, approximate=approximate) * x[..., d:]
|
||||
|
||||
|
||||
@gelu_and_mul_sparse.register_input_generator
|
||||
def _gelu_and_mul_sparse_input_generator(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
std_multiplier: float = 1.6448536269514722,
|
||||
approximate: str = "tanh",
|
||||
) -> tuple:
|
||||
x = torch.randn(num_tokens, hidden_size * 2, dtype=dtype)
|
||||
return x, std_multiplier, approximate
|
||||
Reference in New Issue
Block a user