[Bugfix] Fix MoE model load OOM in FlashInfer_TRTLLM backend with sleep mode (#45589)

Signed-off-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-06-16 19:36:51 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ef2c40dc00
commit 71bc19dbdd
2 changed files with 43 additions and 32 deletions
@@ -269,8 +269,6 @@ def convert_to_unquantized_kernel_format(
w13_weight = swap_w13_to_w31(w13_weight) w13_weight = swap_w13_to_w31(w13_weight)
elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM: elif unquantized_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
# Swap halves to arrange as [w3; w1] (kernel expectation)
w13_weight = swap_w13_to_w31(w13_weight)
_cache_permute_indices: dict[torch.Size, torch.Tensor] = {} _cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
w13_weight, w2_weight = convert_moe_weights_to_flashinfer_trtllm_block_layout( w13_weight, w2_weight = convert_moe_weights_to_flashinfer_trtllm_block_layout(
_cache_permute_indices, _cache_permute_indices,
@@ -121,7 +121,6 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
from flashinfer.fused_moe.core import ( from flashinfer.fused_moe.core import (
_maybe_get_cached_w3_w1_permute_indices, _maybe_get_cached_w3_w1_permute_indices,
convert_to_block_layout,
get_w2_permute_indices_with_cache, get_w2_permute_indices_with_cache,
) )
@@ -131,23 +130,49 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
# Reorder rows of W13 and W2 for fused gated activation and convert to the # Reorder rows of W13 and W2 for fused gated activation and convert to the
# block layout expected by the FlashInfer kernel. # block layout expected by the FlashInfer kernel.
num_experts = w13_weight.shape[0] num_experts = w13_weight.shape[0]
device_w13 = w13_weight.device
device_w2 = w2_weight.device
w13_weights_shuffled: list[torch.Tensor] = [] def _copy_permuted_expert_to_block_layout(
w2_weights_shuffled: list[torch.Tensor] = [] out: torch.Tensor,
expert_uint8: torch.Tensor,
source_indices: torch.Tensor,
) -> None:
expert_blocks = expert_uint8.view(
expert_uint8.shape[0], out.shape[0], block_k
).permute(1, 0, 2)
torch.index_select(
expert_blocks,
1,
source_indices.to(expert_uint8.device),
out=out,
)
w13_rows, w13_cols = w13_weight[0].view(torch.uint8).shape
w2_rows, w2_cols = w2_weight[0].view(torch.uint8).shape
w13_weights_shuffled_tensor = torch.empty(
(num_experts, w13_cols // block_k, w13_rows, block_k),
dtype=torch.uint8,
device=w13_weight.device,
)
w2_weights_shuffled_tensor = torch.empty(
(num_experts, w2_cols // block_k, w2_rows, block_k),
dtype=torch.uint8,
device=w2_weight.device,
)
for i in range(num_experts): for i in range(num_experts):
w13_expert_uint8 = w13_weight[i].view(torch.uint8)
permute_indices = _maybe_get_cached_w3_w1_permute_indices( permute_indices = _maybe_get_cached_w3_w1_permute_indices(
cache_permute_indices, cache_permute_indices,
w13_weight[i].view(torch.uint8), w13_expert_uint8,
epilogue_tile_m, epilogue_tile_m,
) )
tmp_weights1 = ( rows = w13_expert_uint8.shape[0]
w13_weight[i] permute_indices = (permute_indices + rows // 2) % rows
.clone() _copy_permuted_expert_to_block_layout(
.view(torch.uint8)[permute_indices.to(device_w13)] w13_weights_shuffled_tensor[i],
.contiguous() w13_expert_uint8,
permute_indices,
) )
permute_indices = get_w2_permute_indices_with_cache( permute_indices = get_w2_permute_indices_with_cache(
@@ -155,28 +180,16 @@ def convert_moe_weights_to_flashinfer_trtllm_block_layout(
w2_weight[i].view(torch.uint8), w2_weight[i].view(torch.uint8),
epilogue_tile_m, epilogue_tile_m,
) )
tmp_weights2 = ( _copy_permuted_expert_to_block_layout(
w2_weight[i] w2_weights_shuffled_tensor[i],
.clone() w2_weight[i].view(torch.uint8),
.view(torch.uint8)[permute_indices.to(device_w2)] permute_indices,
.contiguous()
) )
tmp_weights1 = convert_to_block_layout(tmp_weights1.view(torch.uint8), block_k) return (
tmp_weights2 = convert_to_block_layout(tmp_weights2.view(torch.uint8), block_k) w13_weights_shuffled_tensor.view(torch.bfloat16),
w2_weights_shuffled_tensor.view(torch.bfloat16),
w13_weights_shuffled.append(tmp_weights1.view(torch.bfloat16))
w2_weights_shuffled.append(tmp_weights2.view(torch.bfloat16))
# Stack weights for all experts and return as BF16 tensors.
w13_weights_shuffled_tensor = (
torch.stack(w13_weights_shuffled).view(torch.bfloat16).contiguous()
) )
w2_weights_shuffled_tensor = (
torch.stack(w2_weights_shuffled).view(torch.bfloat16).contiguous()
)
return w13_weights_shuffled_tensor, w2_weights_shuffled_tensor
def align_fp4_moe_weights_for_fi( def align_fp4_moe_weights_for_fi(