Integrate CuTeDSL MoE for ReLU2 NVFP4 (#49580)
Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Michael Goin
parent
30c2718eaa
commit
32a423ac0a
@@ -103,7 +103,8 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
|
||||
"""Prepare weights for the CuteDSL wrapper-based NvFP4 MoE backend.
|
||||
|
||||
Converts weight scale factors to MMA layout expected by CuteDslMoEWrapper,
|
||||
and interleaves w13 gate/linear rows.
|
||||
and interleaves w13 gate/linear rows for gated activations. Non-gated
|
||||
activations use a single w13 projection and keep its row order unchanged.
|
||||
"""
|
||||
from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout
|
||||
|
||||
@@ -112,13 +113,14 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
|
||||
a13_scale = a13_scale.max().to(torch.float32).repeat(num_experts)
|
||||
a2_scale = a2_scale.max().to(torch.float32).repeat(num_experts)
|
||||
|
||||
half = w13.shape[1] // 2
|
||||
w13 = torch.cat([w13[:, half:], w13[:, :half]], dim=1)
|
||||
w13_scale = torch.cat([w13_scale[:, half:], w13_scale[:, :half]], dim=1)
|
||||
if layer.activation.is_gated:
|
||||
half = w13.shape[1] // 2
|
||||
w13 = torch.cat([w13[:, half:], w13[:, :half]], dim=1)
|
||||
w13_scale = torch.cat([w13_scale[:, half:], w13_scale[:, :half]], dim=1)
|
||||
|
||||
# Interleave up/gate rows for w13 weights and scales.
|
||||
w13 = interleave_linear_and_gate(w13, group_size=64, dim=1)
|
||||
w13_scale = interleave_linear_and_gate(w13_scale, group_size=64, dim=1)
|
||||
# Interleave up/gate rows for w13 weights and scales.
|
||||
w13 = interleave_linear_and_gate(w13, group_size=64, dim=1)
|
||||
w13_scale = interleave_linear_and_gate(w13_scale, group_size=64, dim=1)
|
||||
|
||||
# Convert w13 scale factors: linear → swizzled → MMA layout.
|
||||
w13_scale = swizzle_blockscale(w13_scale)
|
||||
|
||||
Reference in New Issue
Block a user