[Kernel] Add b12x FP4 MoE backend (#52018)
Signed-off-by: Luke Alonso <lalonso@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
+14
-4
@@ -25,9 +25,11 @@ from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
|
||||
XPUExpertsMxFp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
B12X_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
make_mxfp4_moe_kernel,
|
||||
make_mxfp4_moe_quant_config,
|
||||
select_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
|
||||
CompressedTensorsMoEMethod,
|
||||
@@ -46,10 +48,17 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
super().__init__(moe)
|
||||
self.group_size = 32
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
|
||||
# use cutlass if supported, otherwise fallback to marlin for weight-only FP4
|
||||
# Backend selection must match the weight preparation below: CUTLASS
|
||||
# swizzles scales, b12x and XPU consume checkpoint packing, and Marlin
|
||||
# repacks weights and scales.
|
||||
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
|
||||
self.experts_cls: type[mk.FusedMoEExperts]
|
||||
if self.use_cutlass_mxfp4:
|
||||
if moe.moe_backend == "b12x":
|
||||
self.mxfp4_backend, experts_cls = select_mxfp4_moe_backend(moe)
|
||||
assert experts_cls is not None
|
||||
self.experts_cls = experts_cls
|
||||
self.use_cutlass_mxfp4 = False
|
||||
elif self.use_cutlass_mxfp4:
|
||||
logger.info_once("Using CutlassExpertsMxfp4 for MXFP4 MoE")
|
||||
self.experts_cls = CutlassExpertsMxfp4
|
||||
elif current_platform.is_xpu():
|
||||
@@ -138,7 +147,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
)
|
||||
else:
|
||||
# W4A16: weight-only via Marlin
|
||||
# b12x selects W4A8 or W4A16; XPU uses W4A4; Marlin uses W4A16.
|
||||
return make_mxfp4_moe_quant_config(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
@@ -188,7 +197,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer.w2_weight_scale = torch.nn.Parameter(
|
||||
torch.stack(swizzled_w2), requires_grad=False
|
||||
)
|
||||
elif current_platform.is_xpu():
|
||||
elif self.mxfp4_backend in B12X_BACKENDS or current_platform.is_xpu():
|
||||
pass
|
||||
else:
|
||||
logger.warning_once(
|
||||
@@ -208,6 +217,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
|
||||
+3
@@ -52,6 +52,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
):
|
||||
super().__init__(moe)
|
||||
self.group_size = 16
|
||||
self.use_a16 = use_a16
|
||||
|
||||
# Select experts implementation.
|
||||
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
|
||||
@@ -231,6 +232,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
w2_scale_2=(1.0 / layer.w2_weight_global_scale),
|
||||
a2_scale=(1.0 / layer.w2_input_global_scale),
|
||||
is_act_and_mul=self.moe.is_act_and_mul,
|
||||
use_a16=self.use_a16,
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
@@ -267,6 +269,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
swiglu_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
swiglu_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
use_a16=self.use_a16,
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
B12X_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
make_mxfp4_moe_kernel,
|
||||
make_mxfp4_moe_quant_config,
|
||||
@@ -42,26 +43,29 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
"""W4A4 MXFP4 group MoE for AutoRound ``auto_round:llm_compressor`` exports.
|
||||
|
||||
Registers the packed MXFP4 layout (uint8 ``weight_packed`` + uint8 E8M0
|
||||
``weight_scale``, ``group_size=32``) and dispatches the fused MoE to the
|
||||
best backend for the current device: CUTLASS (true W4A4 on supported
|
||||
GPUs), the native XPU kernel, or Marlin weight-only as a fallback. The
|
||||
per-expert ``gate_proj`` / ``up_proj`` / ``down_proj`` tensors are folded
|
||||
into the stacked ``w13`` / ``w2`` parameters by ``make_expert_params_mapping``.
|
||||
``weight_scale``, ``group_size=32``) and dispatches the fused MoE to b12x
|
||||
when requested; otherwise it uses CUTLASS W4A4, XPU W4A4, or Marlin W4A16.
|
||||
The per-expert ``gate_proj`` / ``up_proj`` / ``down_proj`` tensors are
|
||||
folded into the stacked ``w13`` / ``w2`` parameters by
|
||||
``make_expert_params_mapping``.
|
||||
"""
|
||||
|
||||
def __init__(self, moe) -> None:
|
||||
super().__init__(moe)
|
||||
self.group_size = 32
|
||||
# Backend selection must stay consistent with the weight preparation in
|
||||
# process_weights_after_loading / get_fused_moe_quant_config, which only
|
||||
# implement three layouts: CUTLASS swizzle (true W4A4), the native XPU
|
||||
# kernel (packed passthrough), and Marlin weight-only. XPU dispatch is
|
||||
# deferred to the shared oracle; every other non-CUTLASS device falls
|
||||
# back to Marlin (mirroring CompressedTensorsW4A4Mxfp4MoEMethod).
|
||||
# process_weights_after_loading / get_fused_moe_quant_config, which use
|
||||
# three layouts: CUTLASS swizzle (true W4A4), b12x and XPU packed
|
||||
# passthrough, and Marlin weight-only. XPU dispatch is deferred to the
|
||||
# shared oracle; b12x dispatch is explicit; every remaining non-CUTLASS
|
||||
# device falls back to Marlin.
|
||||
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
|
||||
self.experts_cls: type[mk.FusedMoEExperts] | None = None
|
||||
if self.use_cutlass_mxfp4:
|
||||
if moe.moe_backend == "b12x":
|
||||
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
|
||||
self.use_cutlass_mxfp4 = False
|
||||
elif self.use_cutlass_mxfp4:
|
||||
self.experts_cls = CutlassExpertsMxfp4
|
||||
logger.info_once("Using CutlassExpertsMxfp4 for AutoRound MXFP4 MoE")
|
||||
elif current_platform.is_xpu():
|
||||
@@ -146,7 +150,7 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
)
|
||||
# Weight-only (Marlin) or native XPU kernel.
|
||||
# b12x selects W4A8 or W4A16; XPU uses W4A4; Marlin uses W4A16.
|
||||
return make_mxfp4_moe_quant_config(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
@@ -194,9 +198,7 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
layer.w2_weight_scale = torch.nn.Parameter(
|
||||
torch.stack(swizzled_w2), requires_grad=False
|
||||
)
|
||||
elif current_platform.is_xpu():
|
||||
# The XPU fused-MoE kernel consumes the packed layout directly; no
|
||||
# swizzle / repack / transpose is required.
|
||||
elif self.mxfp4_backend in B12X_BACKENDS or current_platform.is_xpu():
|
||||
pass
|
||||
else:
|
||||
logger.warning_once(
|
||||
@@ -216,6 +218,7 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
|
||||
@@ -1574,6 +1574,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
w2_scale_2=layer.w2_weight_scale_2,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
is_act_and_mul=self.moe.is_act_and_mul,
|
||||
use_a16=self.use_a16,
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
@@ -1610,6 +1611,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
swiglu_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
swiglu_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
use_a16=self.use_a16,
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
@@ -376,6 +376,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
w13 = layer.w13_weight
|
||||
@@ -769,6 +770,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
w13 = layer.w13_weight
|
||||
|
||||
@@ -255,6 +255,7 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
|
||||
@@ -1334,6 +1334,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: RoutedExperts
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weight preparation helpers for b12x MoE kernels."""
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
swap_w13_to_w31,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
swizzle_blockscale,
|
||||
)
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
|
||||
def _pad_gated_weights(
|
||||
w13: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
if w13_scale.size(1) != w13.size(1):
|
||||
raise ValueError("w13 weight and scale row counts must match")
|
||||
half_size = w13.size(1) // 2
|
||||
if half_size * 2 != w13.size(1):
|
||||
raise ValueError("gated NVFP4 MoE weights must have even row counts")
|
||||
if w2.size(2) * 2 != half_size:
|
||||
raise ValueError("w2 shape does not match gated w13")
|
||||
if w2_scale.size(2) * 16 != half_size:
|
||||
raise ValueError("w2 scale shape does not match gated w13")
|
||||
|
||||
half_pad_size = round_up(half_size, 64) - half_size
|
||||
if half_pad_size == 0:
|
||||
return w13, w13_scale, w2, w2_scale
|
||||
|
||||
def pad_rows(tensor: torch.Tensor) -> torch.Tensor:
|
||||
shape = tensor.shape
|
||||
return F.pad(
|
||||
tensor.reshape(shape[0], 2, half_size, *shape[2:]),
|
||||
(0, 0, 0, half_pad_size),
|
||||
).flatten(1, 2)
|
||||
|
||||
return (
|
||||
pad_rows(w13),
|
||||
pad_rows(w13_scale),
|
||||
F.pad(w2, (0, half_pad_size // 2)),
|
||||
F.pad(w2_scale, (0, half_pad_size // 16)),
|
||||
)
|
||||
|
||||
|
||||
def _per_expert_scale(
|
||||
scale: torch.Tensor,
|
||||
num_experts: int,
|
||||
name: str,
|
||||
) -> torch.Tensor:
|
||||
if scale.dim() == 0:
|
||||
scale = scale.expand(num_experts)
|
||||
elif scale.dim() == 2:
|
||||
scale = scale.amax(dim=1)
|
||||
if scale.dim() != 1 or scale.numel() != num_experts:
|
||||
raise ValueError(f"{name} must contain one value per expert")
|
||||
return scale.to(torch.float32).contiguous()
|
||||
|
||||
|
||||
def prepare_nvfp4_moe_layer_for_b12x(
|
||||
w13: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w13_scale_2: torch.Tensor,
|
||||
a13_scale: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w2_scale_2: torch.Tensor,
|
||||
a2_scale: torch.Tensor,
|
||||
is_act_and_mul: bool,
|
||||
reorder_w13: bool = False,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
]:
|
||||
"""Prepare b12x NVFP4 MoE weights and scales."""
|
||||
num_experts = w13.shape[0]
|
||||
a13_scale = _per_expert_scale(a13_scale, num_experts, "a13_scale")
|
||||
a2_scale = _per_expert_scale(a2_scale, num_experts, "a2_scale")
|
||||
|
||||
if reorder_w13 and is_act_and_mul:
|
||||
w13 = swap_w13_to_w31(w13)
|
||||
w13_scale = swap_w13_to_w31(w13_scale)
|
||||
if is_act_and_mul:
|
||||
w13, w13_scale, w2, w2_scale = _pad_gated_weights(w13, w13_scale, w2, w2_scale)
|
||||
|
||||
w13_scale = swizzle_blockscale(w13_scale)
|
||||
pad_size = w13_scale.size(1) - w13.size(1)
|
||||
if pad_size > 0:
|
||||
if is_act_and_mul:
|
||||
raise RuntimeError("gated NVFP4 MoE padding must precede scale swizzling")
|
||||
w13 = F.pad(w13, (0, 0, 0, pad_size))
|
||||
w2 = F.pad(w2, (0, pad_size // 2, 0, 0))
|
||||
w2_scale = F.pad(w2_scale, (0, pad_size // 16))
|
||||
|
||||
w2_scale = swizzle_blockscale(w2_scale)
|
||||
return w13, w13_scale, w13_scale_2, a13_scale, w2, w2_scale, w2_scale_2, a2_scale
|
||||
Reference in New Issue
Block a user