[Bugfix][MoE] Convert FlashInfer BF16 weights in place (#54699)
Signed-off-by: Yuchen Wang <93700456+yuchenwang3@users.noreply.github.com> Signed-off-by: Dakai An <dakaian108@gmail.com> Co-authored-by: Dakai An <dakaian108@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
Dakai An
OpenAI Codex
aoshen02
parent
117cf43fb6
commit
78fdf4efb8
@@ -112,8 +112,9 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Convert expert weights to FlashInfer's block layout.
|
||||
|
||||
This reorders W13 and W2 into the expected epilogue-tiled block layout and
|
||||
returns the shuffled weight tensors.
|
||||
This reorders W13 and W2 in place into the expected epilogue-tiled block
|
||||
layout and returns views of the shuffled weight tensors. Using one expert
|
||||
as scratch space avoids allocating another full copy of both weights.
|
||||
"""
|
||||
if w13_weight.dtype != torch.bfloat16 or w2_weight.dtype != torch.bfloat16:
|
||||
raise ValueError(
|
||||
@@ -147,51 +148,50 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
out=out,
|
||||
)
|
||||
|
||||
w13_rows, w13_cols = w13_weight[0].view(torch.uint8).shape
|
||||
w2_rows, w2_cols = w2_weight[0].view(torch.uint8).shape
|
||||
w13_weights_shuffled_tensor = torch.empty(
|
||||
(num_experts, w13_cols // block_k, w13_rows, block_k),
|
||||
dtype=torch.uint8,
|
||||
device=w13_weight.device,
|
||||
)
|
||||
w2_weights_shuffled_tensor = torch.empty(
|
||||
(num_experts, w2_cols // block_k, w2_rows, block_k),
|
||||
dtype=torch.uint8,
|
||||
device=w2_weight.device,
|
||||
)
|
||||
|
||||
for i in range(num_experts):
|
||||
w13_expert_uint8 = w13_weight[i].view(torch.uint8)
|
||||
|
||||
permute_indices = _maybe_get_cached_w3_w1_permute_indices(
|
||||
cache_permute_indices,
|
||||
w13_expert_uint8,
|
||||
epilogue_tile_m,
|
||||
is_gated_act_gemm=is_gated_act_gemm,
|
||||
)
|
||||
if is_gated_act_gemm:
|
||||
rows = w13_expert_uint8.shape[0]
|
||||
permute_indices = (permute_indices + rows // 2) % rows
|
||||
_copy_permuted_expert_to_block_layout(
|
||||
w13_weights_shuffled_tensor[i],
|
||||
w13_expert_uint8,
|
||||
permute_indices,
|
||||
def _convert_weight_in_place(
|
||||
weight: torch.Tensor,
|
||||
is_w13: bool,
|
||||
) -> torch.Tensor:
|
||||
rows, cols = weight[0].view(torch.uint8).shape
|
||||
block_layout_shape = (num_experts, cols // block_k, rows, block_k)
|
||||
expert_scratch = torch.empty(
|
||||
block_layout_shape[1:],
|
||||
dtype=torch.uint8,
|
||||
device=weight.device,
|
||||
)
|
||||
|
||||
permute_indices = get_w2_permute_indices_with_cache(
|
||||
cache_permute_indices,
|
||||
w2_weight[i].view(torch.uint8),
|
||||
epilogue_tile_m,
|
||||
)
|
||||
_copy_permuted_expert_to_block_layout(
|
||||
w2_weights_shuffled_tensor[i],
|
||||
w2_weight[i].view(torch.uint8),
|
||||
permute_indices,
|
||||
)
|
||||
for i in range(num_experts):
|
||||
expert_uint8 = weight[i].view(torch.uint8)
|
||||
if is_w13:
|
||||
permute_indices = _maybe_get_cached_w3_w1_permute_indices(
|
||||
cache_permute_indices,
|
||||
expert_uint8,
|
||||
epilogue_tile_m,
|
||||
is_gated_act_gemm=is_gated_act_gemm,
|
||||
)
|
||||
if is_gated_act_gemm:
|
||||
permute_indices = (
|
||||
permute_indices + expert_uint8.shape[0] // 2
|
||||
) % expert_uint8.shape[0]
|
||||
else:
|
||||
permute_indices = get_w2_permute_indices_with_cache(
|
||||
cache_permute_indices,
|
||||
expert_uint8,
|
||||
epilogue_tile_m,
|
||||
)
|
||||
|
||||
_copy_permuted_expert_to_block_layout(
|
||||
expert_scratch,
|
||||
expert_uint8,
|
||||
permute_indices,
|
||||
)
|
||||
expert_uint8.view(-1).copy_(expert_scratch.view(-1))
|
||||
|
||||
return weight.view(torch.uint8).view(block_layout_shape).view(torch.bfloat16)
|
||||
|
||||
return (
|
||||
w13_weights_shuffled_tensor.view(torch.bfloat16),
|
||||
w2_weights_shuffled_tensor.view(torch.bfloat16),
|
||||
_convert_weight_in_place(w13_weight, is_w13=True),
|
||||
_convert_weight_in_place(w2_weight, is_w13=False),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user