Feature: Enable Flashinfer non-gated MoE bf16 (#43853)

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
This commit is contained in:
amirkl94
2026-06-17 14:32:49 +00:00
committed by GitHub
parent 0b131b16c9
commit 8b2b566ea7
3 changed files with 30 additions and 8 deletions
@@ -108,6 +108,7 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
cache_permute_indices: dict[torch.Size, torch.Tensor],
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
is_gated_act_gemm: bool = True,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Convert expert weights to FlashInfer's block layout.
@@ -166,9 +167,11 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
cache_permute_indices,
w13_expert_uint8,
epilogue_tile_m,
is_gated_act_gemm=is_gated_act_gemm,
)
rows = w13_expert_uint8.shape[0]
permute_indices = (permute_indices + rows // 2) % rows
if is_gated_act_gemm:
rows = w13_expert_uint8.shape[0]
permute_indices = (permute_indices + rows // 2) % rows
_copy_permuted_expert_to_block_layout(
w13_weights_shuffled_tensor[i],
w13_expert_uint8,
@@ -288,12 +291,12 @@ def align_trtllm_fp4_moe_hidden_dim_for_fi(
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_hidden_size
def align_fp8_moe_weights_for_fi(
def align_moe_weights_for_fi(
w13: torch.Tensor, w2: torch.Tensor, is_act_and_mul: bool, min_alignment: int = 16
) -> tuple[torch.Tensor, torch.Tensor, int]:
"""Pad intermediate size so FlashInfer kernels' alignment constraints hold.
Some FlashInfer FP8 MoE kernels require the (gated) intermediate size
Some FlashInfer MoE kernels require the (gated) intermediate size
used for GEMM to be divisible by a small alignment value. When this is
not satisfied (e.g. with certain tensor-parallel sizes), we pad the
gate/up and down projection weights along the intermediate dim.
@@ -492,7 +495,7 @@ def prepare_fp8_moe_layer_for_fi(
# for the gate-up proj. Pad the weights to respect this.
if not block_quant:
min_alignment = 16 if is_gated else 128
w13, w2, new_intermediate = align_fp8_moe_weights_for_fi(
w13, w2, new_intermediate = align_moe_weights_for_fi(
w13,
w2,
layer.moe_config.is_act_and_mul,