[Bugfix][Quantization] Fix MXFP4 conversion for FlashInfer CUTLASS (#51038)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -27,6 +27,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
mxfp4_w4a16_moe_quant_config,
|
||||
ocp_mx_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
swap_w13_to_w31,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import _swizzle_mxfp4
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
OCP_MX_BLOCK_SIZE,
|
||||
@@ -1278,7 +1281,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
]:
|
||||
"""Convert loaded weights into backend-specific kernel format.
|
||||
|
||||
Supports DeepGEMM, TRTLLM MXFP8, Triton and Marlin backends.
|
||||
Supports DeepGEMM, FlashInfer, TRTLLM MXFP8, Triton and Marlin backends.
|
||||
"""
|
||||
from vllm.platforms.rocm import on_gfx1250
|
||||
|
||||
@@ -1569,6 +1572,60 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
elif mxfp4_backend in (
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
):
|
||||
# Standard checkpoints store fused gate/up tensors as [w1; w3], while
|
||||
# FlashInfer CUTLASS consumes [w3; w1]. Keep weights, scales, and bias
|
||||
# in the same order before applying the backend-specific interleave.
|
||||
w13_weight = swap_w13_to_w31(w13_weight.data)
|
||||
w13_weight_scale = swap_w13_to_w31(w13_weight_scale.data)
|
||||
if w13_bias is not None:
|
||||
b1, b3 = torch.chunk(w13_bias.data, 2, dim=-1)
|
||||
w13_bias = torch.cat([b3, b1], dim=-1).to(torch.bfloat16)
|
||||
if w2_bias is not None:
|
||||
w2_bias = w2_bias.data.to(torch.bfloat16)
|
||||
|
||||
if mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8:
|
||||
from flashinfer import block_scale_interleave
|
||||
|
||||
w13_scale_shape = w13_weight_scale.shape
|
||||
w13_weight_scale = block_scale_interleave(
|
||||
w13_weight_scale.view(torch.uint8)
|
||||
).reshape(w13_scale_shape)
|
||||
|
||||
w2_scale_shape = w2_weight_scale.shape
|
||||
w2_weight_scale = block_scale_interleave(
|
||||
w2_weight_scale.data.view(torch.uint8)
|
||||
).reshape(w2_scale_shape)
|
||||
else:
|
||||
from flashinfer.fused_moe import (
|
||||
interleave_moe_scales_for_sm90_mixed_gemm,
|
||||
interleave_moe_weights_for_sm90_mixed_gemm,
|
||||
)
|
||||
|
||||
w13_weight = interleave_moe_weights_for_sm90_mixed_gemm(
|
||||
w13_weight.contiguous(), "fp4"
|
||||
)
|
||||
w2_weight = interleave_moe_weights_for_sm90_mixed_gemm(
|
||||
w2_weight.data.contiguous(), "fp4"
|
||||
)
|
||||
w13_weight_scale = interleave_moe_scales_for_sm90_mixed_gemm(
|
||||
w13_weight_scale.to(torch.uint8)
|
||||
)
|
||||
w2_weight_scale = interleave_moe_scales_for_sm90_mixed_gemm(
|
||||
w2_weight_scale.data.to(torch.uint8)
|
||||
)
|
||||
|
||||
return (
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
w13_weight_scale,
|
||||
w2_weight_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
elif mxfp4_backend in (
|
||||
Mxfp4MoeBackend.XPU,
|
||||
Mxfp4MoeBackend.EMULATION,
|
||||
@@ -1586,7 +1643,8 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported mxfp4_backend for Mxfp4MoEMethod: {mxfp4_backend}. "
|
||||
f"Expected TRTLLM, Triton, AITER, XPU, or emulation backend."
|
||||
"Expected TRTLLM, FlashInfer CUTLASS, Triton, AITER, XPU, or "
|
||||
"emulation backend."
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user