[Kernel] Add MXFP4 W4A4 CUTLASS MoE kernel for SM100 (#37463)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
+65
-13
@@ -4,6 +4,7 @@
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
@@ -11,6 +12,10 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
mxfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.cutlass_moe import (
|
||||
CutlassExpertsMxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_marlin_moe import (
|
||||
MarlinExperts,
|
||||
@@ -36,7 +41,14 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
super().__init__(moe)
|
||||
self.group_size = 32
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
|
||||
self.experts_cls = MarlinExperts
|
||||
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
|
||||
self.experts_cls: type[mk.FusedMoEExperts]
|
||||
if self.use_cutlass_mxfp4:
|
||||
logger.info_once("Using CutlassExpertsMxfp4 for MXFP4 MoE", scope="local")
|
||||
self.experts_cls = CutlassExpertsMxfp4
|
||||
else:
|
||||
logger.info_once("Using MarlinExperts for MXFP4 MoE", scope="local")
|
||||
self.experts_cls = MarlinExperts
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -109,11 +121,19 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
return make_mxfp4_moe_quant_config(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
)
|
||||
if self.use_cutlass_mxfp4:
|
||||
# W4A4: both weights and activations quantized to MXFP4
|
||||
return mxfp4_moe_quant_config(
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
)
|
||||
else:
|
||||
# W4A16: weight-only via Marlin
|
||||
return make_mxfp4_moe_quant_config(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: FusedMoE) -> None:
|
||||
layer.w13_weight = torch.nn.Parameter(
|
||||
@@ -126,13 +146,45 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
)
|
||||
delattr(layer, "w2_weight_packed")
|
||||
|
||||
logger.warning_once(
|
||||
"Your GPU does not have native support for FP4 computation but "
|
||||
"FP4 quantization is being used. Weight-only FP4 compression "
|
||||
"will be used leveraging the Marlin kernel. This may degrade "
|
||||
"performance for compute-heavy workloads."
|
||||
)
|
||||
prepare_moe_fp4_layer_for_marlin(layer)
|
||||
if self.use_cutlass_mxfp4:
|
||||
# Swizzle weight scales from flat checkpoint layout [E, N, K//32]
|
||||
# to CUTLASS tiled layout [E, numMTiles*numKTiles*512].
|
||||
from vllm.model_executor.layers.fused_moe.cutlass_moe import (
|
||||
swizzle_mxfp4_scales,
|
||||
)
|
||||
|
||||
E = layer.w13_weight_scale.shape[0]
|
||||
w13_N = layer.w13_weight_scale.shape[1]
|
||||
w13_scale_K = layer.w13_weight_scale.shape[2]
|
||||
w13_K = w13_scale_K * 32
|
||||
|
||||
w2_M = layer.w2_weight_scale.shape[1]
|
||||
w2_scale_N = layer.w2_weight_scale.shape[2]
|
||||
w2_N = w2_scale_N * 32
|
||||
|
||||
swizzled_w13 = []
|
||||
swizzled_w2 = []
|
||||
for e_idx in range(E):
|
||||
s13 = layer.w13_weight_scale[e_idx]
|
||||
sw13 = swizzle_mxfp4_scales(s13, w13_N, w13_K)
|
||||
swizzled_w13.append(sw13.reshape(w13_N, w13_scale_K))
|
||||
s2 = layer.w2_weight_scale[e_idx]
|
||||
sw2 = swizzle_mxfp4_scales(s2, w2_M, w2_N)
|
||||
swizzled_w2.append(sw2.reshape(w2_M, w2_scale_N))
|
||||
layer.w13_weight_scale = torch.nn.Parameter(
|
||||
torch.stack(swizzled_w13), requires_grad=False
|
||||
)
|
||||
layer.w2_weight_scale = torch.nn.Parameter(
|
||||
torch.stack(swizzled_w2), requires_grad=False
|
||||
)
|
||||
else:
|
||||
logger.warning_once(
|
||||
"Your GPU does not have native support for FP4 computation "
|
||||
"but FP4 quantization is being used. Weight-only FP4 "
|
||||
"compression will be used leveraging the Marlin kernel. "
|
||||
"This may degrade performance for compute-heavy workloads."
|
||||
)
|
||||
prepare_moe_fp4_layer_for_marlin(layer)
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None:
|
||||
|
||||
Reference in New Issue
Block a user