[Performance] Add Triton kernel for Gemma3n sparse GELU (#48498)
Signed-off-by: BWAAEEEK <jooho414@gmail.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Andreas Karatzas
Cyrus Leung
parent
e6b4e47d2d
commit
78e84261ab
@@ -11,6 +11,7 @@ from vllm.compilation.passes.ir.lowering_pass import (
|
||||
)
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.ir import ops
|
||||
from vllm.model_executor.layers.activation import GeluAndMulSparse
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...backend import TestBackend
|
||||
@@ -34,6 +35,15 @@ class Model(nn.Module):
|
||||
return x6 + 3.0
|
||||
|
||||
|
||||
class SparseActivationModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.activation = GeluAndMulSparse(0.95, "tanh")
|
||||
|
||||
def forward(self, x):
|
||||
return self.activation(x)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rms_provider", ops.rms_norm.supported_providers())
|
||||
def test_lowering_rms_norm(rms_provider, default_vllm_config):
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
@@ -67,3 +77,27 @@ def test_lowering_rms_norm(rms_provider, default_vllm_config):
|
||||
|
||||
torch.testing.assert_close(output_unlowered, output)
|
||||
torch.testing.assert_close(output_unlowered, output2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_lowering_gelu_and_mul_sparse(default_vllm_config):
|
||||
torch.set_default_device("cuda")
|
||||
lowering_pass = VllmIRLoweringPass(get_current_vllm_config())
|
||||
backend = TestBackend(lowering_pass)
|
||||
model = SparseActivationModel()
|
||||
x = torch.randn(8, 32768, dtype=torch.bfloat16)
|
||||
|
||||
with (
|
||||
ops.gelu_and_mul_sparse.set_priority(["triton", "native"]),
|
||||
ir.enable_torch_wrap(True),
|
||||
):
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
|
||||
output = compiled_model(x)
|
||||
|
||||
selected = lowering_pass.selected_impls["gelu_and_mul_sparse"]
|
||||
assert selected == {"gelu_and_mul_sparse": "triton"}
|
||||
|
||||
expected = ops.gelu_and_mul_sparse.impls["native"].impl_fn(
|
||||
x, 1.6448536269514722, "tanh"
|
||||
)
|
||||
torch.testing.assert_close(output, expected, rtol=1.6e-2, atol=1e-3)
|
||||
|
||||
Reference in New Issue
Block a user