diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 583378fdb4..207d827f36 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -367,6 +367,7 @@ steps: - csrc/quantization/fp4/ - csrc/libtorch_stable/nvfp4_kv_cache_kernels.cu - csrc/libtorch_stable/quantization/fp4/ + - csrc/libtorch_stable/quantization/w8a8/cutlass/ - csrc/attention/mla/ - csrc/quantization/cutlass_w8a8/moe/ - vllm/model_executor/layers/fusion/ @@ -408,6 +409,7 @@ steps: - tests/kernels/test_ll_bf16_gemm.py - tests/kernels/test_top_k_per_row.py - tests/kernels/quantization/test_silu_mul_nvfp4_quant.py + - tests/kernels/quantization/test_block_fp8.py - tests/models/quantization/test_nvfp4.py commands: - nvidia-smi @@ -442,6 +444,7 @@ steps: pytest -v -s tests/kernels/attention/test_flashinfer.py -k 'not num_heads2' pytest -v -s tests/kernels/attention/test_cutlass_mla_decode.py pytest -v -s tests/kernels/quantization/test_cutlass_scaled_mm.py -k 'fp8' + pytest -v -s tests/kernels/quantization/test_block_fp8.py::test_w8a8_block_fp8_cutlass_matmul pytest -v -s tests/kernels/quantization/test_nvfp4_quant.py pytest -v -s tests/kernels/quantization/test_nvfp4_scaled_mm.py pytest -v -s tests/kernels/quantization/test_mxfp4_qutlass.py diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8_dispatch.cuh b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8_dispatch.cuh index 8f74f29914..fa2add9182 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8_dispatch.cuh +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8_dispatch.cuh @@ -152,6 +152,29 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te int32_t m = a.size(0), n = b.size(1), k = a.size(1); + // The SM100 blockwise SF copy requires the SFA M extent to be a multiple + // of 4 (4-row copy atom). For misaligned M on the non-swapAB path, pad only + // the per-token activation scales (column-major (m, k/128), 1/64 the size + // of the bf16 activation) and build layout_SFA from the padded extent. + int32_t m_sf = m; + std::optional a_scales_pad; + void const* a_scales_data = a_scales.data_ptr(); + if constexpr (!swap_ab) { + if (m % 4 != 0) { + m_sf = (m + 3) & ~3; + int64_t num_groups = a_scales.numel() / m; + a_scales_pad = torch::stable::new_empty( + a_scales, {num_groups * m_sf}, + torch::headeronly::ScalarType::Float); + cudaMemcpy2DAsync(a_scales_pad->data_ptr(), m_sf * sizeof(float), + a_scales.data_ptr(), m * sizeof(float), + m * sizeof(float), num_groups, + cudaMemcpyDeviceToDevice, + get_current_cuda_stream(a.get_device())); + a_scales_data = a_scales_pad->data_ptr(); + } + } + StrideA a_stride; StrideB b_stride; StrideC c_stride; @@ -162,16 +185,16 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te c_stride = cutlass::make_cute_packed_stride(StrideC{}, swap_ab ? cute::make_shape(n, m, 1) : cute::make_shape(m, n, 1)); - LayoutSFA layout_SFA = swap_ab ? + LayoutSFA layout_SFA = swap_ab ? ScaleConfig::tile_atom_to_shape_SFA(make_shape(n, m, k, 1)) : - ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1)); + ScaleConfig::tile_atom_to_shape_SFA(make_shape(m_sf, n, k, 1)); LayoutSFB layout_SFB = swap_ab ? ScaleConfig::tile_atom_to_shape_SFB(make_shape(n, m, k, 1)) : ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1)); auto a_ptr = static_cast(a.data_ptr()); auto b_ptr = static_cast(b.data_ptr()); - auto a_scales_ptr = static_cast(a_scales.data_ptr()); + auto a_scales_ptr = static_cast(a_scales_data); auto b_scales_ptr = static_cast(b_scales.data_ptr()); typename GemmKernel::MainloopArguments mainloop_args{}; @@ -212,7 +235,9 @@ void cutlass_gemm_blockwise_sm100_fp8_dispatch(torch::stable::Tensor& out, constexpr int TILE_K = 128; // TODO: better heuristics - bool swap_ab = (m < 16) || (m % 4 != 0); + // swapAB only wins for small M; misaligned M >= 65 is faster on the + // non-swapAB path with padded activation scales (measured on GB300). + bool swap_ab = (m <= 64); bool use_tma_epilogue = (m * n) % 4 == 0; if (!swap_ab) { constexpr int TILE_N = 128; diff --git a/tests/kernels/quantization/test_block_fp8.py b/tests/kernels/quantization/test_block_fp8.py index d02527aba0..038b94bd2d 100644 --- a/tests/kernels/quantization/test_block_fp8.py +++ b/tests/kernels/quantization/test_block_fp8.py @@ -162,11 +162,16 @@ def test_w8a8_block_fp8_matmul(M, N, K, block_size, out_dtype, seed): @pytest.mark.skipif( not current_platform.is_cuda(), reason="CUTLASS only supported on CUDA platform." ) +@pytest.mark.parametrize( + # 65/66/67 cover all M%4 residue classes above the SM100 swapAB + # threshold (m <= 64); 1026 crosses multiple 128-row SF atoms. + "M", + [32, 65, 66, 67, 1026], +) @torch.inference_mode() -def test_w8a8_block_fp8_cutlass_matmul(): +def test_w8a8_block_fp8_cutlass_matmul(M): # Test simple case where weight.shape % 128 != 0, # like in DSV3 kv_a_proj_with_mqa - M = 32 N = 576 K = 7168 block_size = [128, 128]