[Perf][DSv4] Use native CUDA SwiGLU clamp kernel for Humming MoE (throughput +1.40%) (#53685)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>

tests/kernels/core/test_activation.py

@@ -22,6 +22,7 @@ from vllm.model_executor.layers.activation import (
     SwigluStepAndMul,
     swiglustep_and_mul_triton,
 )
+from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func
 from vllm.utils.torch_utils import set_random_seed
 
 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -121,6 +122,20 @@ def test_act_and_mul(
 SWIGLU_LIMITS = [3.0, 7.0, 15.0]
 
 
+@torch.inference_mode()
+def test_swiglu_limit_func_without_routing_uses_output_buffer() -> None:
+    x = torch.randn(7, 1024, dtype=torch.bfloat16, device="cuda")
+    output = torch.empty(7, 512, dtype=x.dtype, device=x.device)
+
+    swiglu_limit_func(output, x, swiglu_limit=7.0)
+    gate, up = x.chunk(2, dim=-1)
+    expected = torch.nn.functional.silu(gate.clamp(max=7.0)) * up.clamp(
+        min=-7.0, max=7.0
+    )
+
+    torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2)
+
+
 @pytest.mark.parametrize("swiglu_limit", SWIGLU_LIMITS)
 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
 @pytest.mark.parametrize("d", D)

vllm/model_executor/layers/fused_moe/utils.py

@@ -647,6 +647,8 @@ def swiglu_limit_func(
     # requires topk_ids. Fall back to the torch implementation otherwise.
     if topk_ids is not None:
         _swiglu_limit_pad_aware(output, input, topk_ids, swiglu_limit, expert_map)
+    elif current_platform.is_cuda():
+        torch.ops._C.silu_and_mul_with_clamp(output, input, swiglu_limit, 1.0, 0.0)
     else:
         _swiglu_limit_torch(output, input, swiglu_limit)