[Bugfix][MoE] Convert FlashInfer BF16 weights in place (#54699)

Signed-off-by: Yuchen Wang <93700456+yuchenwang3@users.noreply.github.com>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Yuchen Wang
2026-09-17 22:00:04 +08:00
committed by GitHub
co-authored by Dakai An OpenAI Codex aoshen02
parent 117cf43fb6
commit 78fdf4efb8
9 changed files with 667 additions and 68 deletions
@@ -112,8 +112,9 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
) -> tuple[torch.Tensor, torch.Tensor]:
"""Convert expert weights to FlashInfer's block layout.
This reorders W13 and W2 into the expected epilogue-tiled block layout and
returns the shuffled weight tensors.
This reorders W13 and W2 in place into the expected epilogue-tiled block
layout and returns views of the shuffled weight tensors. Using one expert
as scratch space avoids allocating another full copy of both weights.
"""
if w13_weight.dtype != torch.bfloat16 or w2_weight.dtype != torch.bfloat16:
raise ValueError(
@@ -147,51 +148,50 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
out=out,
)
w13_rows, w13_cols = w13_weight[0].view(torch.uint8).shape
w2_rows, w2_cols = w2_weight[0].view(torch.uint8).shape
w13_weights_shuffled_tensor = torch.empty(
(num_experts, w13_cols // block_k, w13_rows, block_k),
dtype=torch.uint8,
device=w13_weight.device,
)
w2_weights_shuffled_tensor = torch.empty(
(num_experts, w2_cols // block_k, w2_rows, block_k),
dtype=torch.uint8,
device=w2_weight.device,
)
for i in range(num_experts):
w13_expert_uint8 = w13_weight[i].view(torch.uint8)
permute_indices = _maybe_get_cached_w3_w1_permute_indices(
cache_permute_indices,
w13_expert_uint8,
epilogue_tile_m,
is_gated_act_gemm=is_gated_act_gemm,
)
if is_gated_act_gemm:
rows = w13_expert_uint8.shape[0]
permute_indices = (permute_indices + rows // 2) % rows
_copy_permuted_expert_to_block_layout(
w13_weights_shuffled_tensor[i],
w13_expert_uint8,
permute_indices,
def _convert_weight_in_place(
weight: torch.Tensor,
is_w13: bool,
) -> torch.Tensor:
rows, cols = weight[0].view(torch.uint8).shape
block_layout_shape = (num_experts, cols // block_k, rows, block_k)
expert_scratch = torch.empty(
block_layout_shape[1:],
dtype=torch.uint8,
device=weight.device,
)
permute_indices = get_w2_permute_indices_with_cache(
cache_permute_indices,
w2_weight[i].view(torch.uint8),
epilogue_tile_m,
)
_copy_permuted_expert_to_block_layout(
w2_weights_shuffled_tensor[i],
w2_weight[i].view(torch.uint8),
permute_indices,
)
for i in range(num_experts):
expert_uint8 = weight[i].view(torch.uint8)
if is_w13:
permute_indices = _maybe_get_cached_w3_w1_permute_indices(
cache_permute_indices,
expert_uint8,
epilogue_tile_m,
is_gated_act_gemm=is_gated_act_gemm,
)
if is_gated_act_gemm:
permute_indices = (
permute_indices + expert_uint8.shape[0] // 2
) % expert_uint8.shape[0]
else:
permute_indices = get_w2_permute_indices_with_cache(
cache_permute_indices,
expert_uint8,
epilogue_tile_m,
)
_copy_permuted_expert_to_block_layout(
expert_scratch,
expert_uint8,
permute_indices,
)
expert_uint8.view(-1).copy_(expert_scratch.view(-1))
return weight.view(torch.uint8).view(block_layout_shape).view(torch.bfloat16)
return (
w13_weights_shuffled_tensor.view(torch.bfloat16),
w2_weights_shuffled_tensor.view(torch.bfloat16),
_convert_weight_in_place(w13_weight, is_w13=True),
_convert_weight_in_place(w2_weight, is_w13=False),
)