From 71b961dd356a399150d25738c175c71859aa1301 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Sat, 13 Jun 2026 15:05:45 -0400 Subject: [PATCH] [Perf] SM90 cutlass fp8 mm supports odd M by swap_ab, 180~290% kernel performance improvement (#44572) Signed-off-by: yewentao256 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../scaled_mm_blockwise_sm90_fp8_dispatch.cuh | 126 ++++++++++++------ .../quantization/test_cutlass_scaled_mm.py | 2 - .../kernels/linear/scaled_mm/cutlass.py | 118 ---------------- 3 files changed, 87 insertions(+), 159 deletions(-) diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm90_fp8_dispatch.cuh b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm90_fp8_dispatch.cuh index cf62e81fd7..529b28ceec 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm90_fp8_dispatch.cuh +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm90_fp8_dispatch.cuh @@ -25,33 +25,43 @@ using namespace cute; template + class EpilogueScheduler, class MainloopScheduler, + bool swap_ab_ = false> struct cutlass_3x_gemm_fp8_blockwise { + static constexpr bool swap_ab = swap_ab_; using ElementAB = cutlass::float_e4m3_t; using ElementA = ElementAB; using LayoutA = cutlass::layout::RowMajor; + using LayoutA_Transpose = typename cutlass::layout::LayoutTranspose::type; static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; using ElementB = ElementAB; using LayoutB = cutlass::layout::ColumnMajor; + using LayoutB_Transpose = typename cutlass::layout::LayoutTranspose::type; static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; using ElementD = OutType; using LayoutD = cutlass::layout::RowMajor; + using LayoutD_Transpose = typename cutlass::layout::LayoutTranspose::type; static constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; using ElementC = void; // TODO: support bias using LayoutC = LayoutD; + using LayoutC_Transpose = LayoutD_Transpose; static constexpr int AlignmentC = AlignmentD; using ElementAccumulator = float; using ElementCompute = float; using ElementBlockScale = float; - using ScaleConfig = cutlass::detail::Sm90BlockwiseScaleConfig< + using ScaleConfig = conditional_t; + cute::GMMA::Major::K, cute::GMMA::Major::MN>, + cutlass::detail::Sm90BlockwiseScaleConfig< + ScaleGranularityM, ScaleGranularityN, ScaleGranularityK, + cute::GMMA::Major::MN, cute::GMMA::Major::K>>; using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); @@ -71,30 +81,46 @@ struct cutlass_3x_gemm_fp8_blockwise { ElementAccumulator, ElementCompute, ElementC, - LayoutC, + conditional_t, AlignmentC, ElementD, - LayoutD, + conditional_t, AlignmentD, EpilogueScheduler, DefaultOperation >::CollectiveOp; - using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< - ArchTag, - OperatorClass, - ElementA, - cute::tuple, - AlignmentA, - ElementB, - cute::tuple, - AlignmentB, - ElementAccumulator, - MmaTileShape, - ClusterShape, - cutlass::gemm::collective::StageCountAutoCarveout(sizeof(typename CollectiveEpilogue::SharedStorage))>, - MainloopScheduler - >::CollectiveOp; + using CollectiveMainloop = conditional_t, + AlignmentB, + ElementA, + cute::tuple, + AlignmentA, + ElementAccumulator, + MmaTileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout(sizeof(typename CollectiveEpilogue::SharedStorage))>, + MainloopScheduler + >::CollectiveOp, + typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + cute::tuple, + AlignmentA, + ElementB, + cute::tuple, + AlignmentB, + ElementAccumulator, + MmaTileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout(sizeof(typename CollectiveEpilogue::SharedStorage))>, + MainloopScheduler + >::CollectiveOp>; using KernelType = enable_sm90_or_later, CollectiveMainloop, CollectiveEpilogue>>; @@ -107,6 +133,7 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te torch::stable::Tensor const& b, torch::stable::Tensor const& a_scales, torch::stable::Tensor const& b_scales) { + static constexpr bool swap_ab = Gemm::swap_ab; using GemmKernel = typename Gemm::GemmKernel; using StrideA = typename Gemm::GemmKernel::StrideA; using StrideB = typename Gemm::GemmKernel::StrideB; @@ -122,8 +149,6 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te int32_t m = a.size(0), n = b.size(1), k = a.size(1); - STD_TORCH_CHECK(m % 4 == 0, "m must be divisible by 4"); - StrideA a_stride; StrideB b_stride; StrideC c_stride; @@ -132,12 +157,16 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te b_stride = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(n, k, 1)); c_stride = - cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(m, n, 1)); + cutlass::make_cute_packed_stride( + StrideC{}, swap_ab ? cute::make_shape(n, m, 1) + : cute::make_shape(m, n, 1)); - LayoutSFA layout_SFA = - ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1)); - LayoutSFB layout_SFB = - ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1)); + LayoutSFA layout_SFA = swap_ab + ? ScaleConfig::tile_atom_to_shape_SFA(make_shape(n, m, k, 1)) + : ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1)); + LayoutSFB layout_SFB = swap_ab + ? ScaleConfig::tile_atom_to_shape_SFB(make_shape(n, m, k, 1)) + : ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1)); auto a_ptr = static_cast(a.data_ptr()); auto b_ptr = static_cast(b.data_ptr()); @@ -145,15 +174,25 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te auto b_scales_ptr = static_cast(b_scales.data_ptr()); typename GemmKernel::MainloopArguments mainloop_args{}; - mainloop_args.ptr_A = a_ptr; - mainloop_args.dA = a_stride; - mainloop_args.ptr_B = b_ptr; - mainloop_args.dB = b_stride; - mainloop_args.ptr_SFA = a_scales_ptr; mainloop_args.layout_SFA = layout_SFA; - mainloop_args.ptr_SFB = b_scales_ptr; mainloop_args.layout_SFB = layout_SFB; - auto prob_shape = cute::make_shape(m, n, k, 1); + if (swap_ab) { + mainloop_args.ptr_A = b_ptr; + mainloop_args.dA = b_stride; + mainloop_args.ptr_B = a_ptr; + mainloop_args.dB = a_stride; + mainloop_args.ptr_SFA = b_scales_ptr; + mainloop_args.ptr_SFB = a_scales_ptr; + } else { + mainloop_args.ptr_A = a_ptr; + mainloop_args.dA = a_stride; + mainloop_args.ptr_B = b_ptr; + mainloop_args.dB = b_stride; + mainloop_args.ptr_SFA = a_scales_ptr; + mainloop_args.ptr_SFB = b_scales_ptr; + } + auto prob_shape = swap_ab ? cute::make_shape(n, m, k, 1) + : cute::make_shape(m, n, k, 1); auto c_ptr = static_cast(out.data_ptr()); typename GemmKernel::EpilogueArguments epilogue_args{ @@ -168,12 +207,21 @@ void cutlass_gemm_blockwise_sm90_fp8_dispatch(torch::stable::Tensor& out, torch::stable::Tensor const& b, torch::stable::Tensor const& a_scales, torch::stable::Tensor const& b_scales) { - // TODO: better heuristics + bool swap_ab = (a.size(0) % 4) != 0; + if (!swap_ab) { + cutlass_gemm_caller_blockwise, + Shape<_1, _2, _1>, cutlass::epilogue::TmaWarpSpecializedCooperative, + cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8BlockScaledAccum>>( + out, a, b, a_scales, b_scales); + return; + } + cutlass_gemm_caller_blockwise, - Shape<_1, _2, _1>, cutlass::epilogue::TmaWarpSpecializedCooperative, - cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8BlockScaledAccum>>( - out, a, b, a_scales, b_scales); + OutType, 128, 1, 128, Shape<_128, _16, _128>, + Shape<_1, _1, _1>, cutlass::epilogue::TmaWarpSpecialized, + cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8BlockScaledAccum, + true>>(out, a, b, a_scales, b_scales); } } // namespace vllm \ No newline at end of file diff --git a/tests/kernels/quantization/test_cutlass_scaled_mm.py b/tests/kernels/quantization/test_cutlass_scaled_mm.py index a937c30fed..25893311af 100644 --- a/tests/kernels/quantization/test_cutlass_scaled_mm.py +++ b/tests/kernels/quantization/test_cutlass_scaled_mm.py @@ -245,8 +245,6 @@ def test_cutlass_fp8_blockwise_scale_gemm( return if m % a_scale_group_shape[0] != 0 or k % a_scale_group_shape[1] != 0: return - if m % 4 != 0 and current_platform.has_device_capability(100): - return cutlass_fp8_gemm_helper(m, n, k, a_scale_group_shape, b_scale_group_shape, use_bias) diff --git a/vllm/model_executor/kernels/linear/scaled_mm/cutlass.py b/vllm/model_executor/kernels/linear/scaled_mm/cutlass.py index 7e25541f17..9f69ab0c73 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/cutlass.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/cutlass.py @@ -20,7 +20,6 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( ) from vllm.model_executor.utils import set_weight_attrs from vllm.platforms import current_platform -from vllm.utils.torch_utils import direct_register_custom_op from .BlockScaledMMLinearKernel import Fp8BlockScaledMMLinearKernel from .ScaledMMLinearKernel import ( @@ -277,7 +276,6 @@ class CutlassFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel): def __init__(self, config: FP8ScaledMMLinearLayerConfig) -> None: super().__init__(config) act_scale_descriptor = config.activation_quant_key.scale - self.weight_group_shape = config.weight_quant_key.scale.group_shape self.quant_fp8 = QuantFP8( static=act_scale_descriptor.static, group_shape=act_scale_descriptor.group_shape, @@ -285,7 +283,6 @@ class CutlassFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel): use_ue8m0=False, column_major_scales=True, ) - self.is_hopper = current_platform.is_device_capability(90) @classmethod def is_supported(cls, compute_capability=None): @@ -320,16 +317,6 @@ class CutlassFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel): Bs: torch.Tensor, ) -> torch.Tensor: out_dtype = self.config.out_dtype - if self.is_hopper: - return torch.ops.vllm.dynamic_padded_cutlass( - A, - B, - As, - Bs, - list(self.weight_group_shape), - out_dtype, - ) - return ops.cutlass_scaled_mm( A, B.T, @@ -354,108 +341,3 @@ def cutlass_scaled_mm( scale_a=As, scale_b=Bs.T, ) - - -def _padded_cutlass( - qx: torch.Tensor, - weight: torch.Tensor, - x_scale: torch.Tensor, - weight_scale: torch.Tensor, - block_size: list[int], - output_dtype: torch.dtype, -) -> torch.Tensor: - pad_multiple = 4 - dim = qx.shape[0] - padded = ( - dim if dim % pad_multiple == 0 else dim + pad_multiple - (dim % pad_multiple) - ) - - has_pad = padded > dim - - if has_pad: - padded_shape = [padded, *qx.shape[1:]] - padded_qx = torch.zeros(padded_shape, device=qx.device, dtype=qx.dtype) - padded_qx[0 : qx.shape[0], ...].copy_(qx) - - padded_x_scale_shape = [*x_scale.shape[1:], padded] - padded_x_scale = torch.ones( - padded_x_scale_shape, device=x_scale.device, dtype=x_scale.dtype - ).permute(-1, -2) - padded_x_scale[0 : x_scale.shape[0], ...].copy_(x_scale) - - output = cutlass_scaled_mm( - padded_qx, weight, padded_x_scale, weight_scale, block_size, output_dtype - ) - return output[0 : qx.shape[0], ...] - else: - return cutlass_scaled_mm( - qx, weight, x_scale, weight_scale, block_size, output_dtype - ) - - -def _padded_cutlass_fake( - qx: torch.Tensor, - weight: torch.Tensor, - x_scale: torch.Tensor, - weight_scale: torch.Tensor, - block_size: list[int], - output_dtype: torch.dtype, -) -> torch.Tensor: - return torch.empty( - (qx.size(0), weight.size(0)), dtype=output_dtype, device=qx.device - ) - - -def _dynamic_padded_cutlass( - qx: torch.Tensor, - weight: torch.Tensor, - x_scale: torch.Tensor, - weight_scale: torch.Tensor, - block_size: list[int], - output_dtype: torch.dtype, -) -> torch.Tensor: - def run_padded( - qx: torch.Tensor, - weight: torch.Tensor, - x_scale: torch.Tensor, - weight_scale: torch.Tensor, - ) -> torch.Tensor: - return _padded_cutlass( - qx, weight, x_scale, weight_scale, block_size, output_dtype - ) - - def run_direct( - qx: torch.Tensor, - weight: torch.Tensor, - x_scale: torch.Tensor, - weight_scale: torch.Tensor, - ) -> torch.Tensor: - return cutlass_scaled_mm( - qx, weight, x_scale, weight_scale, block_size, output_dtype - ) - - if torch.compiler.is_compiling(): - return torch.cond( - qx.shape[0] % 4 != 0, - run_padded, - run_direct, - (qx, weight, x_scale, weight_scale), - ) - - if qx.shape[0] % 4 != 0: - return run_padded(qx, weight, x_scale, weight_scale) - - return run_direct(qx, weight, x_scale, weight_scale) - - -direct_register_custom_op( - "padded_cutlass", - _padded_cutlass, - fake_impl=_padded_cutlass_fake, -) - -direct_register_custom_op( - "dynamic_padded_cutlass", - _dynamic_padded_cutlass, - fake_impl=_padded_cutlass_fake, -)