[Bugfix] Fix Kimi K3 NVFP4 MoE weight conversion OOM (#55407)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
This commit is contained in:
Wei Zhao
2026-09-07 11:45:25 +08:00
committed by GitHub
parent c3ec0d29f5
commit 6748217fb9
@@ -190,25 +190,23 @@ def nvfp4_swizzled_scale_to_cutedsl_mma_view(scale: torch.Tensor) -> torch.Tenso
def prepare_static_weights_for_trtllm_fp4_moe( def prepare_static_weights_for_trtllm_fp4_moe(
# args_dequant, gemm1_weights: torch.Tensor,
# args, gemm2_weights: torch.Tensor,
gemm1_weights, gemm1_scales_linear_fp4_bytes: torch.Tensor,
gemm2_weights, gemm2_scales_linear_fp4_bytes: torch.Tensor,
gemm1_scales_linear_fp4_bytes, hidden_size: int,
gemm2_scales_linear_fp4_bytes, intermediate_size: int,
hidden_size, num_experts: int,
intermediate_size,
num_experts,
is_gated_activation: bool, is_gated_activation: bool,
): ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Shuffle NVFP4 weights into the FlashInfer TRT-LLM layout."""
from flashinfer import nvfp4_block_scale_interleave from flashinfer import nvfp4_block_scale_interleave
from flashinfer.fused_moe.core import ( from flashinfer.fused_moe.core import (
_maybe_get_cached_w3_w1_permute_indices, _maybe_get_cached_w3_w1_permute_indices,
get_w2_permute_indices_with_cache, get_w2_permute_indices_with_cache,
) )
_cache_permute_indices: dict[torch.Size, torch.Tensor] = {} permute_indices_cache: dict[tuple[object, ...], torch.Tensor] = {}
"""Prepare quantized weights for kernel (done offline with weights)."""
epilogue_tile_m = 128 # FIXME: this depends on the kernel internals epilogue_tile_m = 128 # FIXME: this depends on the kernel internals
gemm1_intermediate_size = ( gemm1_intermediate_size = (
2 * intermediate_size if is_gated_activation else intermediate_size 2 * intermediate_size if is_gated_activation else intermediate_size
@@ -231,85 +229,75 @@ def prepare_static_weights_for_trtllm_fp4_moe(
torch.float8_e4m3fn torch.float8_e4m3fn
).reshape(num_experts, hidden_size, intermediate_size // 16) # fp8 scaling factors ).reshape(num_experts, hidden_size, intermediate_size // 16) # fp8 scaling factors
gemm1_weights_fp4_shuffled = [] gemm1_weight_indices = _maybe_get_cached_w3_w1_permute_indices(
gemm1_scales_fp4_shuffled = [] permute_indices_cache,
gemm2_weights_fp4_shuffled = [] gemm1_weights_fp4[0].view(torch.uint8),
gemm2_scales_fp4_shuffled = [] epilogue_tile_m,
for i in range(num_experts): is_gated_act_gemm=is_gated_activation,
# Calculate the permute indices for the following: )
# 1. Reorder rows of W1 and scales for fused gated activation gemm1_scale_indices = _maybe_get_cached_w3_w1_permute_indices(
# 2. Shuffle weights and scaling factors for transposed mma output permute_indices_cache,
# for both w3_w1 and w2 weights and scale factors gemm1_scales_linear_fp4[0].view(torch.uint8),
permute_indices = _maybe_get_cached_w3_w1_permute_indices( epilogue_tile_m,
_cache_permute_indices, num_elts_per_sf=16,
gemm1_weights_fp4[i].view(torch.uint8), is_gated_act_gemm=is_gated_activation,
epilogue_tile_m, )
is_gated_act_gemm=is_gated_activation, gemm2_weight_indices = get_w2_permute_indices_with_cache(
) permute_indices_cache,
gemm1_weights_fp4_shuffled.append( gemm2_weights_fp4[0].view(torch.uint8),
gemm1_weights_fp4[i] epilogue_tile_m,
.view(torch.uint8)[permute_indices.to(gemm1_weights_fp4.device)] )
.contiguous() gemm2_scale_indices = get_w2_permute_indices_with_cache(
) permute_indices_cache,
gemm2_scales_linear_fp4[0].view(torch.uint8),
permute_sf_indices = _maybe_get_cached_w3_w1_permute_indices( epilogue_tile_m,
_cache_permute_indices, num_elts_per_sf=16,
gemm1_scales_linear_fp4[i].view(torch.uint8),
epilogue_tile_m,
num_elts_per_sf=16,
is_gated_act_gemm=is_gated_activation,
)
gemm1_scales_fp4_shuffled.append(
nvfp4_block_scale_interleave(
gemm1_scales_linear_fp4[i]
.view(torch.uint8)[
permute_sf_indices.to(gemm1_scales_linear_fp4.device)
]
.contiguous()
)
)
permute_indices = get_w2_permute_indices_with_cache(
_cache_permute_indices,
gemm2_weights_fp4[i].view(torch.uint8),
epilogue_tile_m,
)
gemm2_weights_fp4_shuffled.append(
gemm2_weights_fp4[i]
.view(torch.uint8)[permute_indices.to(gemm2_weights_fp4.device)]
.contiguous()
)
permute_sf_indices = get_w2_permute_indices_with_cache(
_cache_permute_indices,
gemm2_scales_linear_fp4[i].view(torch.uint8),
epilogue_tile_m,
num_elts_per_sf=16,
)
gemm2_scales_fp4_shuffled.append(
nvfp4_block_scale_interleave(
gemm2_scales_linear_fp4[i]
.view(torch.uint8)[
permute_sf_indices.to(gemm2_scales_linear_fp4.device)
]
.contiguous()
)
)
# Stack weights for all experts
gemm1_weights_fp4_shuffled = torch.stack(gemm1_weights_fp4_shuffled)
gemm1_scales_fp4_shuffled = (
torch.stack(gemm1_scales_fp4_shuffled)
.view(torch.float8_e4m3fn)
.reshape(num_experts, gemm1_intermediate_size, hidden_size // 16)
) )
gemm2_weights_fp4_shuffled = torch.stack(gemm2_weights_fp4_shuffled) gemm1_weights_fp4_shuffled = torch.empty_like(gemm1_weights_fp4, dtype=torch.uint8)
gemm2_scales_fp4_shuffled = ( gemm1_scales_fp4_shuffled = torch.empty_like(gemm1_scales_linear_fp4)
torch.stack(gemm2_scales_fp4_shuffled) gemm2_weights_fp4_shuffled = torch.empty_like(gemm2_weights_fp4, dtype=torch.uint8)
.view(torch.float8_e4m3fn) gemm2_scales_fp4_shuffled = torch.empty_like(gemm2_scales_linear_fp4)
.reshape(num_experts, hidden_size, intermediate_size // 16) gemm1_scale_scratch = torch.empty_like(
gemm1_scales_linear_fp4[0], dtype=torch.uint8
) )
gemm2_scale_scratch = torch.empty_like(
gemm2_scales_linear_fp4[0], dtype=torch.uint8
)
for expert_id in range(num_experts):
torch.index_select(
gemm1_weights_fp4[expert_id].view(torch.uint8),
0,
gemm1_weight_indices,
out=gemm1_weights_fp4_shuffled[expert_id],
)
torch.index_select(
gemm1_scales_linear_fp4[expert_id].view(torch.uint8),
0,
gemm1_scale_indices,
out=gemm1_scale_scratch,
)
gemm1_scales_fp4_shuffled[expert_id].view(torch.uint8).reshape(-1).copy_(
nvfp4_block_scale_interleave(gemm1_scale_scratch)
)
torch.index_select(
gemm2_weights_fp4[expert_id].view(torch.uint8),
0,
gemm2_weight_indices,
out=gemm2_weights_fp4_shuffled[expert_id],
)
torch.index_select(
gemm2_scales_linear_fp4[expert_id].view(torch.uint8),
0,
gemm2_scale_indices,
out=gemm2_scale_scratch,
)
gemm2_scales_fp4_shuffled[expert_id].view(torch.uint8).reshape(-1).copy_(
nvfp4_block_scale_interleave(gemm2_scale_scratch)
)
return ( return (
gemm1_weights_fp4_shuffled, gemm1_weights_fp4_shuffled,
gemm1_scales_fp4_shuffled, gemm1_scales_fp4_shuffled,