[Performance] Add Triton kernel for Gemma3n sparse GELU (#48498)
Signed-off-by: BWAAEEEK <jooho414@gmail.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Andreas Karatzas
Cyrus Leung
parent
e6b4e47d2d
commit
78e84261ab
@@ -11,6 +11,7 @@ from vllm.compilation.passes.ir.lowering_pass import (
|
||||
)
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.ir import ops
|
||||
from vllm.model_executor.layers.activation import GeluAndMulSparse
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...backend import TestBackend
|
||||
@@ -34,6 +35,15 @@ class Model(nn.Module):
|
||||
return x6 + 3.0
|
||||
|
||||
|
||||
class SparseActivationModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.activation = GeluAndMulSparse(0.95, "tanh")
|
||||
|
||||
def forward(self, x):
|
||||
return self.activation(x)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rms_provider", ops.rms_norm.supported_providers())
|
||||
def test_lowering_rms_norm(rms_provider, default_vllm_config):
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
@@ -67,3 +77,27 @@ def test_lowering_rms_norm(rms_provider, default_vllm_config):
|
||||
|
||||
torch.testing.assert_close(output_unlowered, output)
|
||||
torch.testing.assert_close(output_unlowered, output2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_lowering_gelu_and_mul_sparse(default_vllm_config):
|
||||
torch.set_default_device("cuda")
|
||||
lowering_pass = VllmIRLoweringPass(get_current_vllm_config())
|
||||
backend = TestBackend(lowering_pass)
|
||||
model = SparseActivationModel()
|
||||
x = torch.randn(8, 32768, dtype=torch.bfloat16)
|
||||
|
||||
with (
|
||||
ops.gelu_and_mul_sparse.set_priority(["triton", "native"]),
|
||||
ir.enable_torch_wrap(True),
|
||||
):
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
|
||||
output = compiled_model(x)
|
||||
|
||||
selected = lowering_pass.selected_impls["gelu_and_mul_sparse"]
|
||||
assert selected == {"gelu_and_mul_sparse": "triton"}
|
||||
|
||||
expected = ops.gelu_and_mul_sparse.impls["native"].impl_fn(
|
||||
x, 1.6448536269514722, "tanh"
|
||||
)
|
||||
torch.testing.assert_close(output, expected, rtol=1.6e-2, atol=1e-3)
|
||||
|
||||
@@ -13,6 +13,8 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
import vllm.kernels # noqa: F401 to register kernels
|
||||
from vllm import ir
|
||||
from vllm.compilation.backends import VllmBackend
|
||||
from vllm.compilation.caching import (
|
||||
StandaloneCompiledArtifacts,
|
||||
@@ -28,6 +30,9 @@ from vllm.config import (
|
||||
)
|
||||
from vllm.envs import disable_envs_cache
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.ir import ops
|
||||
from vllm.model_executor.layers.activation import GeluAndMulSparse
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
|
||||
from ..utils import create_new_process_for_each_test
|
||||
@@ -77,6 +82,18 @@ class CompiledModTuple(torch.nn.Module):
|
||||
return reference_fn_tuple(x)
|
||||
|
||||
|
||||
@support_torch_compile(dynamic_arg_dims={"x": 0})
|
||||
class SparseActivationAotModel(torch.nn.Module):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__()
|
||||
self.up_proj = torch.nn.Linear(32, 256, bias=False)
|
||||
self.activation = GeluAndMulSparse(0.95, "tanh")
|
||||
self.down_proj = torch.nn.Linear(128, 16, bias=False)
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
return self.down_proj(self.activation(self.up_proj(x)))
|
||||
|
||||
|
||||
def make_vllm_config() -> VllmConfig:
|
||||
return VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
@@ -175,6 +192,54 @@ def test_save_and_load(monkeypatch: pytest.MonkeyPatch):
|
||||
assert torch.allclose(ret, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not is_torch_equal_or_newer("2.10.0"),
|
||||
reason="requires CUDA and torch 2.10",
|
||||
)
|
||||
def test_sparse_activation_standalone_aot_preserves_dependencies(
|
||||
monkeypatch: pytest.MonkeyPatch, vllm_tmp_cache: Path
|
||||
):
|
||||
def native_reference(model: SparseActivationAotModel, x: torch.Tensor):
|
||||
projected = torch.nn.functional.linear(x, model.up_proj.weight)
|
||||
activated = ops.gelu_and_mul_sparse.impls["native"].impl_fn(
|
||||
projected, model.activation.std_multiplier, "tanh"
|
||||
)
|
||||
return torch.nn.functional.linear(activated, model.down_proj.weight)
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_USE_AOT_COMPILE", "1")
|
||||
m.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", "1")
|
||||
m.setenv("VLLM_USE_STANDALONE_COMPILE", "1")
|
||||
m.setenv("VLLM_CACHE_ROOT", str(vllm_tmp_cache / "vllm_cache"))
|
||||
|
||||
vllm_config = make_vllm_config()
|
||||
torch.manual_seed(0)
|
||||
inputs = [
|
||||
torch.randn(4, 32, device="cuda", dtype=torch.bfloat16),
|
||||
torch.randn(7, 32, device="cuda", dtype=torch.bfloat16),
|
||||
]
|
||||
|
||||
with (
|
||||
use_vllm_config(vllm_config),
|
||||
vllm_config.kernel_config.ir_op_priority.set_priority(),
|
||||
ir.enable_torch_wrap(True),
|
||||
):
|
||||
compiled = SparseActivationAotModel(vllm_config=vllm_config).to(
|
||||
device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
compiled.requires_grad_(False)
|
||||
expected = [native_reference(compiled, x) for x in inputs]
|
||||
actual = [compiled(x) for x in inputs]
|
||||
|
||||
for result, reference in zip(actual, expected):
|
||||
assert torch.isfinite(result).all()
|
||||
torch.testing.assert_close(
|
||||
result,
|
||||
reference,
|
||||
**ops.gelu_and_mul_sparse.get_tolerance(result.dtype),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
|
||||
def test_save_and_load_slice(monkeypatch: pytest.MonkeyPatch):
|
||||
from torch._subclasses import FakeTensorMode
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.kernels # noqa: F401
|
||||
from tests.ir.ir_test_utils import assert_close, clone_args
|
||||
from vllm import ir
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
gelu_and_mul_sparse_native = ir.ops.gelu_and_mul_sparse.impls["native"].impl_fn
|
||||
|
||||
|
||||
def test_gelu_and_mul_sparse_registration():
|
||||
expected = {
|
||||
"native": True,
|
||||
"triton": HAS_TRITON and current_platform.is_cuda(),
|
||||
}
|
||||
actual = {
|
||||
provider: impl.supported
|
||||
for provider, impl in ir.ops.gelu_and_mul_sparse.impls.items()
|
||||
}
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 8, 64])
|
||||
@pytest.mark.parametrize("intermediate_size", [2048, 4096, 8192, 13824, 16384, 32768])
|
||||
def test_gelu_and_mul_sparse_triton(
|
||||
dtype: torch.dtype, num_tokens: int, intermediate_size: int
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
torch.set_default_device("cuda")
|
||||
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
|
||||
num_tokens=num_tokens,
|
||||
hidden_size=intermediate_size,
|
||||
dtype=dtype,
|
||||
)
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
assert impl.supports_args(*args)
|
||||
|
||||
expected = gelu_and_mul_sparse_native(*clone_args(args))
|
||||
actual = impl.impl_fn(*clone_args(args))
|
||||
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
|
||||
|
||||
mask_mismatch = torch.count_nonzero((actual == 0) != (expected == 0))
|
||||
assert mask_mismatch / actual.numel() < 1e-3
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_second_seed():
|
||||
torch.manual_seed(1)
|
||||
torch.set_default_device("cuda")
|
||||
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
|
||||
num_tokens=32,
|
||||
hidden_size=8192,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*clone_args(args))
|
||||
expected = gelu_and_mul_sparse_native(*clone_args(args))
|
||||
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("intermediate_size", [1, 31, 128])
|
||||
def test_gelu_and_mul_sparse_triton_small_sizes(
|
||||
dtype: torch.dtype, intermediate_size: int
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
torch.set_default_device("cuda")
|
||||
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
|
||||
num_tokens=5,
|
||||
hidden_size=intermediate_size,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*clone_args(args))
|
||||
expected = gelu_and_mul_sparse_native(*clone_args(args))
|
||||
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
|
||||
assert torch.equal(actual == 0, expected == 0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_nd_and_empty():
|
||||
torch.set_default_device("cuda")
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
|
||||
x = torch.randn(2, 3, 16384, dtype=torch.bfloat16)
|
||||
args = (x, 1.6448536269514722, "tanh")
|
||||
actual = impl.impl_fn(*args)
|
||||
expected = gelu_and_mul_sparse_native(*args)
|
||||
assert actual.shape == (2, 3, 8192)
|
||||
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
|
||||
|
||||
empty = torch.empty(0, 16384, dtype=torch.bfloat16)
|
||||
empty_out = impl.impl_fn(empty, 1.6448536269514722, "tanh")
|
||||
assert empty_out.shape == (0, 8192)
|
||||
assert empty_out.dtype == empty.dtype
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
def test_gelu_and_mul_sparse_triton_constant_rows(dtype: torch.dtype):
|
||||
torch.set_default_device("cuda")
|
||||
gate = torch.full((4, 8192), 3.0, dtype=dtype)
|
||||
up = torch.randn_like(gate)
|
||||
x = torch.cat((gate, up), dim=-1)
|
||||
args = (x, 1.6448536269514722, "tanh")
|
||||
|
||||
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*args)
|
||||
expected = gelu_and_mul_sparse_native(*args)
|
||||
torch.testing.assert_close(actual, expected, rtol=0.0, atol=0.0)
|
||||
assert torch.count_nonzero(actual) == 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
@pytest.mark.parametrize("nonfinite", [float("nan"), float("inf")])
|
||||
def test_gelu_and_mul_sparse_triton_nonfinite_gate(nonfinite: float):
|
||||
torch.set_default_device("cuda")
|
||||
x = torch.randn(2, 16384, dtype=torch.bfloat16)
|
||||
x[0, 0] = nonfinite
|
||||
args = (x, 1.6448536269514722, "tanh")
|
||||
|
||||
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*args)
|
||||
expected = gelu_and_mul_sparse_native(*args)
|
||||
torch.testing.assert_close(actual, expected, equal_nan=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_fallback_conditions():
|
||||
torch.set_default_device("cuda")
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
x = torch.randn(4, 16384, dtype=torch.bfloat16)
|
||||
|
||||
assert impl.supports_args(x, 1.0, "tanh")
|
||||
assert not impl.supports_args(x, 1.0, "none")
|
||||
assert not impl.supports_args(x[:, ::2], 1.0, "tanh")
|
||||
assert not impl.supports_args(x[:, :-1], 1.0, "tanh")
|
||||
assert not impl.supports_args(
|
||||
torch.randn(1, 65538, dtype=torch.bfloat16), 1.0, "tanh"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_dispatch_and_opcheck():
|
||||
torch.set_default_device("cuda")
|
||||
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
|
||||
num_tokens=8,
|
||||
hidden_size=4096,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
|
||||
with ir.ops.gelu_and_mul_sparse.set_priority(["triton", "native"]):
|
||||
dispatched = ir.ops.gelu_and_mul_sparse(*args)
|
||||
torch.library.opcheck(torch.ops.vllm_ir.gelu_and_mul_sparse, args)
|
||||
|
||||
direct = impl.impl_fn(*args)
|
||||
torch.testing.assert_close(dispatched, direct, rtol=0.0, atol=0.0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_cudagraph():
|
||||
torch.set_default_device("cuda")
|
||||
x = torch.randn(16, 32768, dtype=torch.bfloat16)
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
|
||||
for _ in range(3):
|
||||
expected = impl.impl_fn(x, 1.6448536269514722, "tanh")
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = impl.impl_fn(x, 1.6448536269514722, "tanh")
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0.0, atol=0.0)
|
||||
@@ -3322,6 +3322,22 @@ def test_ir_op_priority_default():
|
||||
assert priority_config.fused_add_rms_norm == ["native"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", [CompilationMode.NONE, CompilationMode.VLLM_COMPILE])
|
||||
@pytest.mark.parametrize("backend", ["inductor", "eager"])
|
||||
def test_ir_op_platform_defaults_support_sparse_gelu(mode, backend):
|
||||
"""Worker initialization must not select an unregistered sparse GELU provider."""
|
||||
from vllm import ir
|
||||
|
||||
config = SimpleNamespace(
|
||||
compilation_config=CompilationConfig(mode=mode, backend=backend)
|
||||
)
|
||||
priority = current_platform.get_default_ir_op_priority(config)
|
||||
expected = ["triton", "native"] if current_platform.is_cuda() else ["native"]
|
||||
|
||||
with priority.set_priority():
|
||||
assert ir.ops.gelu_and_mul_sparse.get_priority() == expected
|
||||
|
||||
|
||||
def test_ir_op_priority_str():
|
||||
"""Test that passing a comma-delimited string works"""
|
||||
from vllm.config.kernel import IrOpPriorityConfig
|
||||
|
||||
Reference in New Issue
Block a user