[Kernel] Support Nvfp4 Cutedsl Moe Swiglu-oai and Relu2(non-gated) Activation (#47106)

Signed-off-by: Tiekai Bi <tiekaib@nvidia.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Bi Tiekai
2026-08-06 17:42:45 -07:00
committed by GitHub
co-authored by Claude Opus 5 Yongye Zhu
parent b1e12d142d
commit e08111211b
9 changed files with 216 additions and 14 deletions
@@ -0,0 +1,43 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Weight-layout normalization for the FlashInfer CuTeDSL NVFP4 MoE backend."""
import pytest
import torch
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
reorder_w13_to_w31_for_flashinfer_cutedsl,
)
_GATE = torch.tensor([[[1], [2], [3], [4]]])
_UP = torch.tensor([[[10], [20], [30], [40]]])
_EXPECTED = torch.cat([_UP, _GATE], dim=1)
def test_reorder_w13_swigluoai_interleaved():
"""gpt-oss w13 is [gate0, up0, gate1, ...] rather than packed [gate; up]."""
w13 = torch.empty(1, 8, 1, dtype=_GATE.dtype)
w13[:, 0::2] = _GATE
w13[:, 1::2] = _UP
out, out_scale = reorder_w13_to_w31_for_flashinfer_cutedsl(
MoEActivation.SWIGLUOAI, w13, w13 + 100
)
torch.testing.assert_close(out, _EXPECTED)
torch.testing.assert_close(out_scale, _EXPECTED + 100)
@pytest.mark.parametrize(
"activation", [MoEActivation.SILU, MoEActivation.SWIGLUOAI_UNINTERLEAVE]
)
def test_reorder_w13_packed_layouts(activation: MoEActivation):
w13 = torch.cat([_GATE, _UP], dim=1)
out, out_scale = reorder_w13_to_w31_for_flashinfer_cutedsl(
activation, w13, w13 + 100
)
torch.testing.assert_close(out, _EXPECTED)
torch.testing.assert_close(out_scale, _EXPECTED + 100)
@@ -13,9 +13,13 @@ from tests.kernels.quantization.nvfp4_utils import (
FLOAT8_E4M3_MAX,
break_fp4_bytes,
)
from tests.kernels.utils import torch_moe
from vllm import _custom_ops as ops
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.activation import (
SiluAndMul,
SiluAndMulWithClamp,
SwigluOAIAndMul,
)
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
@@ -46,6 +50,77 @@ if not has_flashinfer_cutedsl_moe_nvfp4() or not (
allow_module_level=True,
)
_SWIGLU_ALPHA = 1.702
_SWIGLU_BETA = 1.0
# The GEMM1 output is O(0.5) at these tensor scales, so the OAI default of 7.0
# would never clamp anything. Pick a limit that actually bites.
_SWIGLU_LIMIT = 0.3
_ACT_CASES = [
pytest.param(MoEActivation.SILU, None, None, None, id="silu"),
pytest.param(MoEActivation.SILU, None, None, _SWIGLU_LIMIT, id="silu-clamp"),
pytest.param(MoEActivation.RELU2_NO_MUL, None, None, None, id="relu2_no_mul"),
pytest.param(
MoEActivation.SWIGLUOAI,
_SWIGLU_ALPHA,
_SWIGLU_BETA,
_SWIGLU_LIMIT,
id="swigluoai",
),
pytest.param(
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
_SWIGLU_ALPHA,
_SWIGLU_BETA,
_SWIGLU_LIMIT,
id="swigluoai_uninterleave",
),
]
def _reference_activation(
activation: MoEActivation,
alpha: float | None,
beta: float | None,
limit: float | None,
):
"""vLLM's own op for this activation, so the reference is not re-derived."""
if activation == MoEActivation.RELU2_NO_MUL:
return lambda x: torch.square(torch.relu(x))
if activation == MoEActivation.SWIGLUOAI:
# SwigluOAIAndMul hardcodes beta=1 and reads gate/up interleaved.
assert beta == 1.0
return SwigluOAIAndMul(alpha=alpha, limit=limit)
if activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
return SiluAndMulWithClamp(limit, alpha, beta, compile_native=False)
if limit is not None:
return SiluAndMulWithClamp(limit, compile_native=False)
return SiluAndMul()
def _torch_moe_reference(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
score: torch.Tensor,
topk: int,
act_fn,
) -> torch.Tensor:
m = a.shape[0]
weights, ids = torch.topk(torch.softmax(score, dim=-1, dtype=torch.float32), topk)
x = a.view(m, 1, -1).repeat(1, topk, 1).reshape(m * topk, -1)
flat_ids = ids.reshape(-1)
out = torch.zeros(m * topk, w2.shape[1], dtype=a.dtype, device=a.device)
for expert in range(w1.shape[0]):
mask = flat_ids == expert
if mask.any():
acc = act_fn(x[mask] @ w1[expert].transpose(0, 1)).to(a.dtype)
out[mask] = acc @ w2[expert].transpose(0, 1)
return (
(out.view(m, topk, -1).float() * weights.view(m, topk, 1))
.sum(dim=1)
.to(a.dtype)
)
def _quantize_nvfp4_linear(
weight: torch.Tensor,
@@ -85,14 +160,19 @@ def _dequantize_nvfp4_linear(
@pytest.mark.parametrize("m,n,k,e,topk", [(16, 128, 512, 4, 2)])
@pytest.mark.parametrize("activation,alpha,beta,limit", _ACT_CASES)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@torch.inference_mode()
def test_flashinfer_cutedsl_fp4_moe_relu2_no_mul(
def test_flashinfer_cutedsl_fp4_moe(
m: int,
n: int,
k: int,
e: int,
topk: int,
activation: MoEActivation,
alpha: float | None,
beta: float | None,
limit: float | None,
dtype: torch.dtype,
workspace_init,
):
@@ -102,7 +182,8 @@ def test_flashinfer_cutedsl_fp4_moe_relu2_no_mul(
):
hidden_states = torch.randn((m, k), device="cuda", dtype=dtype) / 10
w1 = torch.randn((e, n, k), device="cuda", dtype=dtype) / 15
w1_rows = 2 * n if activation.is_gated else n
w1 = torch.randn((e, w1_rows, k), device="cuda", dtype=dtype) / 15
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 15
w1_q, w1_scale, w1_global_scale = _quantize_nvfp4_linear(w1)
w2_q, w2_scale, w2_global_scale = _quantize_nvfp4_linear(w2)
@@ -112,7 +193,6 @@ def test_flashinfer_cutedsl_fp4_moe_relu2_no_mul(
hidden_states, score, topk, renormalize=False
)
activation = MoEActivation.RELU2_NO_MUL
fake_layer = SimpleNamespace(activation=activation)
a1_scale = torch.ones(1, device="cuda", dtype=torch.float32)
a2_scale = torch.ones(1, device="cuda", dtype=torch.float32)
@@ -144,6 +224,11 @@ def test_flashinfer_cutedsl_fp4_moe_relu2_no_mul(
w1_scale=w1_scale_cutedsl,
w2_scale=w2_scale_cutedsl,
is_scale_swizzled=False,
# Unset params must be omitted rather than forwarded as None into
# the kernel's float-typed SwiGLU arguments.
gemm1_alpha=alpha,
gemm1_beta=beta,
gemm1_clamp_limit=limit,
)
moe_config = FusedMoEConfig(
num_experts=e,
@@ -198,7 +283,7 @@ def test_flashinfer_cutedsl_fp4_moe_relu2_no_mul(
dtype=dtype,
)
w1_d = torch.empty((e, n, k), device="cuda", dtype=dtype)
w1_d = torch.empty((e, w1_rows, k), device="cuda", dtype=dtype)
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
for idx in range(e):
w1_d[idx] = _dequantize_nvfp4_linear(
@@ -214,17 +299,25 @@ def test_flashinfer_cutedsl_fp4_moe_relu2_no_mul(
dtype=dtype,
)
torch_output = torch_moe(
torch_output = _torch_moe_reference(
a_in_dtype,
w1_d,
w2_d,
score,
topk,
activation=activation,
_reference_activation(activation, alpha, beta, limit),
)
torch.testing.assert_close(
torch_output,
cutedsl_output,
atol=2e-1,
atol=3e-2,
rtol=2e-1,
)
# Outputs here are O(1e-2) while NVFP4 noise is O(1e-3), so an absolute
# tolerance loose enough for the quantization error also accepts a zero
# tensor. Compare direction too, which dropped SwiGLU params or a wrong
# w13 layout would break.
cosine = torch.nn.functional.cosine_similarity(
cutedsl_output.flatten().float(), torch_output.flatten().float(), dim=0
)
assert cosine > 0.99, f"cosine similarity {cosine:.4f} below 0.99"