[Performance] Add Triton kernel for Gemma3n sparse GELU (#48498)

Signed-off-by: BWAAEEEK <jooho414@gmail.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
JooHo Lee
2026-09-14 07:42:15 +00:00
committed by GitHub
co-authored by Andreas Karatzas Cyrus Leung
parent e6b4e47d2d
commit 78e84261ab
15 changed files with 555 additions and 21 deletions
+34
View File
@@ -11,6 +11,7 @@ from vllm.compilation.passes.ir.lowering_pass import (
)
from vllm.config import get_current_vllm_config
from vllm.ir import ops
from vllm.model_executor.layers.activation import GeluAndMulSparse
from vllm.platforms import current_platform
from ...backend import TestBackend
@@ -34,6 +35,15 @@ class Model(nn.Module):
return x6 + 3.0
class SparseActivationModel(nn.Module):
def __init__(self):
super().__init__()
self.activation = GeluAndMulSparse(0.95, "tanh")
def forward(self, x):
return self.activation(x)
@pytest.mark.parametrize("rms_provider", ops.rms_norm.supported_providers())
def test_lowering_rms_norm(rms_provider, default_vllm_config):
torch.set_default_device(current_platform.device_type)
@@ -67,3 +77,27 @@ def test_lowering_rms_norm(rms_provider, default_vllm_config):
torch.testing.assert_close(output_unlowered, output)
torch.testing.assert_close(output_unlowered, output2)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
def test_lowering_gelu_and_mul_sparse(default_vllm_config):
torch.set_default_device("cuda")
lowering_pass = VllmIRLoweringPass(get_current_vllm_config())
backend = TestBackend(lowering_pass)
model = SparseActivationModel()
x = torch.randn(8, 32768, dtype=torch.bfloat16)
with (
ops.gelu_and_mul_sparse.set_priority(["triton", "native"]),
ir.enable_torch_wrap(True),
):
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
output = compiled_model(x)
selected = lowering_pass.selected_impls["gelu_and_mul_sparse"]
assert selected == {"gelu_and_mul_sparse": "triton"}
expected = ops.gelu_and_mul_sparse.impls["native"].impl_fn(
x, 1.6448536269514722, "tanh"
)
torch.testing.assert_close(output, expected, rtol=1.6e-2, atol=1e-3)
+65
View File
@@ -13,6 +13,8 @@ import pytest
import torch
import vllm.envs as envs
import vllm.kernels # noqa: F401 to register kernels
from vllm import ir
from vllm.compilation.backends import VllmBackend
from vllm.compilation.caching import (
StandaloneCompiledArtifacts,
@@ -28,6 +30,9 @@ from vllm.config import (
)
from vllm.envs import disable_envs_cache
from vllm.forward_context import set_forward_context
from vllm.ir import ops
from vllm.model_executor.layers.activation import GeluAndMulSparse
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_torch_equal_or_newer
from ..utils import create_new_process_for_each_test
@@ -77,6 +82,18 @@ class CompiledModTuple(torch.nn.Module):
return reference_fn_tuple(x)
@support_torch_compile(dynamic_arg_dims={"x": 0})
class SparseActivationAotModel(torch.nn.Module):
def __init__(self, **kwargs):
super().__init__()
self.up_proj = torch.nn.Linear(32, 256, bias=False)
self.activation = GeluAndMulSparse(0.95, "tanh")
self.down_proj = torch.nn.Linear(128, 16, bias=False)
def forward(self, x: torch.Tensor):
return self.down_proj(self.activation(self.up_proj(x)))
def make_vllm_config() -> VllmConfig:
return VllmConfig(
compilation_config=CompilationConfig(
@@ -175,6 +192,54 @@ def test_save_and_load(monkeypatch: pytest.MonkeyPatch):
assert torch.allclose(ret, expected)
@pytest.mark.skipif(
not current_platform.is_cuda() or not is_torch_equal_or_newer("2.10.0"),
reason="requires CUDA and torch 2.10",
)
def test_sparse_activation_standalone_aot_preserves_dependencies(
monkeypatch: pytest.MonkeyPatch, vllm_tmp_cache: Path
):
def native_reference(model: SparseActivationAotModel, x: torch.Tensor):
projected = torch.nn.functional.linear(x, model.up_proj.weight)
activated = ops.gelu_and_mul_sparse.impls["native"].impl_fn(
projected, model.activation.std_multiplier, "tanh"
)
return torch.nn.functional.linear(activated, model.down_proj.weight)
with monkeypatch.context() as m:
m.setenv("VLLM_USE_AOT_COMPILE", "1")
m.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", "1")
m.setenv("VLLM_USE_STANDALONE_COMPILE", "1")
m.setenv("VLLM_CACHE_ROOT", str(vllm_tmp_cache / "vllm_cache"))
vllm_config = make_vllm_config()
torch.manual_seed(0)
inputs = [
torch.randn(4, 32, device="cuda", dtype=torch.bfloat16),
torch.randn(7, 32, device="cuda", dtype=torch.bfloat16),
]
with (
use_vllm_config(vllm_config),
vllm_config.kernel_config.ir_op_priority.set_priority(),
ir.enable_torch_wrap(True),
):
compiled = SparseActivationAotModel(vllm_config=vllm_config).to(
device="cuda", dtype=torch.bfloat16
)
compiled.requires_grad_(False)
expected = [native_reference(compiled, x) for x in inputs]
actual = [compiled(x) for x in inputs]
for result, reference in zip(actual, expected):
assert torch.isfinite(result).all()
torch.testing.assert_close(
result,
reference,
**ops.gelu_and_mul_sparse.get_tolerance(result.dtype),
)
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
def test_save_and_load_slice(monkeypatch: pytest.MonkeyPatch):
from torch._subclasses import FakeTensorMode