From 7d5769b2d9cf1e999e3f2fc2c1da265cf0d5d794 Mon Sep 17 00:00:00 2001 From: Chauncey Date: Fri, 28 Aug 2026 04:52:45 +0800 Subject: [PATCH] [Perf][DSv4] Use native CUDA SwiGLU clamp kernel for Humming MoE (throughput +1.40%) (#53685) Signed-off-by: chaunceyjiang --- tests/kernels/core/test_activation.py | 15 +++++++++++++++ vllm/model_executor/layers/fused_moe/utils.py | 2 ++ 2 files changed, 17 insertions(+) diff --git a/tests/kernels/core/test_activation.py b/tests/kernels/core/test_activation.py index f6fc0ab267..f13deccafd 100644 --- a/tests/kernels/core/test_activation.py +++ b/tests/kernels/core/test_activation.py @@ -22,6 +22,7 @@ from vllm.model_executor.layers.activation import ( SwigluStepAndMul, swiglustep_and_mul_triton, ) +from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func from vllm.utils.torch_utils import set_random_seed DTYPES = [torch.half, torch.bfloat16, torch.float] @@ -121,6 +122,20 @@ def test_act_and_mul( SWIGLU_LIMITS = [3.0, 7.0, 15.0] +@torch.inference_mode() +def test_swiglu_limit_func_without_routing_uses_output_buffer() -> None: + x = torch.randn(7, 1024, dtype=torch.bfloat16, device="cuda") + output = torch.empty(7, 512, dtype=x.dtype, device=x.device) + + swiglu_limit_func(output, x, swiglu_limit=7.0) + gate, up = x.chunk(2, dim=-1) + expected = torch.nn.functional.silu(gate.clamp(max=7.0)) * up.clamp( + min=-7.0, max=7.0 + ) + + torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2) + + @pytest.mark.parametrize("swiglu_limit", SWIGLU_LIMITS) @pytest.mark.parametrize("num_tokens", NUM_TOKENS) @pytest.mark.parametrize("d", D) diff --git a/vllm/model_executor/layers/fused_moe/utils.py b/vllm/model_executor/layers/fused_moe/utils.py index 7a4c7fae5a..e1ec850e6a 100644 --- a/vllm/model_executor/layers/fused_moe/utils.py +++ b/vllm/model_executor/layers/fused_moe/utils.py @@ -647,6 +647,8 @@ def swiglu_limit_func( # requires topk_ids. Fall back to the torch implementation otherwise. if topk_ids is not None: _swiglu_limit_pad_aware(output, input, topk_ids, swiglu_limit, expert_map) + elif current_platform.is_cuda(): + torch.ops._C.silu_and_mul_with_clamp(output, input, swiglu_limit, 1.0, 0.0) else: _swiglu_limit_torch(output, input, swiglu_limit)