Integrate CuTeDSL MoE for ReLU2 NVFP4 (#49580)

Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
danielafrimi
2026-07-28 19:46:38 -07:00
committed by GitHub
co-authored by OpenAI Codex Michael Goin
parent 30c2718eaa
commit 32a423ac0a
3 changed files with 245 additions and 9 deletions
@@ -103,7 +103,8 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
"""Prepare weights for the CuteDSL wrapper-based NvFP4 MoE backend.
Converts weight scale factors to MMA layout expected by CuteDslMoEWrapper,
and interleaves w13 gate/linear rows.
and interleaves w13 gate/linear rows for gated activations. Non-gated
activations use a single w13 projection and keep its row order unchanged.
"""
from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout
@@ -112,13 +113,14 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
a13_scale = a13_scale.max().to(torch.float32).repeat(num_experts)
a2_scale = a2_scale.max().to(torch.float32).repeat(num_experts)
half = w13.shape[1] // 2
w13 = torch.cat([w13[:, half:], w13[:, :half]], dim=1)
w13_scale = torch.cat([w13_scale[:, half:], w13_scale[:, :half]], dim=1)
if layer.activation.is_gated:
half = w13.shape[1] // 2
w13 = torch.cat([w13[:, half:], w13[:, :half]], dim=1)
w13_scale = torch.cat([w13_scale[:, half:], w13_scale[:, :half]], dim=1)
# Interleave up/gate rows for w13 weights and scales.
w13 = interleave_linear_and_gate(w13, group_size=64, dim=1)
w13_scale = interleave_linear_and_gate(w13_scale, group_size=64, dim=1)
# Interleave up/gate rows for w13 weights and scales.
w13 = interleave_linear_and_gate(w13, group_size=64, dim=1)
w13_scale = interleave_linear_and_gate(w13_scale, group_size=64, dim=1)
# Convert w13 scale factors: linear → swizzled → MMA layout.
w13_scale = swizzle_blockscale(w13_scale)