C3x SM100 FP8 blockwise - Pad activation scales to multiple of 4 (#57377)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Thien Tran
2026-09-18 08:39:27 +00:00
committed by GitHub
co-authored by Codex
parent 95d723700b
commit 8da75d6190
3 changed files with 39 additions and 6 deletions
+3
View File
@@ -367,6 +367,7 @@ steps:
- csrc/quantization/fp4/
- csrc/libtorch_stable/nvfp4_kv_cache_kernels.cu
- csrc/libtorch_stable/quantization/fp4/
- csrc/libtorch_stable/quantization/w8a8/cutlass/
- csrc/attention/mla/
- csrc/quantization/cutlass_w8a8/moe/
- vllm/model_executor/layers/fusion/
@@ -408,6 +409,7 @@ steps:
- tests/kernels/test_ll_bf16_gemm.py
- tests/kernels/test_top_k_per_row.py
- tests/kernels/quantization/test_silu_mul_nvfp4_quant.py
- tests/kernels/quantization/test_block_fp8.py
- tests/models/quantization/test_nvfp4.py
commands:
- nvidia-smi
@@ -442,6 +444,7 @@ steps:
pytest -v -s tests/kernels/attention/test_flashinfer.py -k 'not num_heads2'
pytest -v -s tests/kernels/attention/test_cutlass_mla_decode.py
pytest -v -s tests/kernels/quantization/test_cutlass_scaled_mm.py -k 'fp8'
pytest -v -s tests/kernels/quantization/test_block_fp8.py::test_w8a8_block_fp8_cutlass_matmul
pytest -v -s tests/kernels/quantization/test_nvfp4_quant.py
pytest -v -s tests/kernels/quantization/test_nvfp4_scaled_mm.py
pytest -v -s tests/kernels/quantization/test_mxfp4_qutlass.py
@@ -152,6 +152,29 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te
int32_t m = a.size(0), n = b.size(1), k = a.size(1);
// The SM100 blockwise SF copy requires the SFA M extent to be a multiple
// of 4 (4-row copy atom). For misaligned M on the non-swapAB path, pad only
// the per-token activation scales (column-major (m, k/128), 1/64 the size
// of the bf16 activation) and build layout_SFA from the padded extent.
int32_t m_sf = m;
std::optional<torch::stable::Tensor> a_scales_pad;
void const* a_scales_data = a_scales.data_ptr();
if constexpr (!swap_ab) {
if (m % 4 != 0) {
m_sf = (m + 3) & ~3;
int64_t num_groups = a_scales.numel() / m;
a_scales_pad = torch::stable::new_empty(
a_scales, {num_groups * m_sf},
torch::headeronly::ScalarType::Float);
cudaMemcpy2DAsync(a_scales_pad->data_ptr(), m_sf * sizeof(float),
a_scales.data_ptr(), m * sizeof(float),
m * sizeof(float), num_groups,
cudaMemcpyDeviceToDevice,
get_current_cuda_stream(a.get_device()));
a_scales_data = a_scales_pad->data_ptr();
}
}
StrideA a_stride;
StrideB b_stride;
StrideC c_stride;
@@ -162,16 +185,16 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te
c_stride =
cutlass::make_cute_packed_stride(StrideC{}, swap_ab ? cute::make_shape(n, m, 1) : cute::make_shape(m, n, 1));
LayoutSFA layout_SFA = swap_ab ?
LayoutSFA layout_SFA = swap_ab ?
ScaleConfig::tile_atom_to_shape_SFA(make_shape(n, m, k, 1)) :
ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1));
ScaleConfig::tile_atom_to_shape_SFA(make_shape(m_sf, n, k, 1));
LayoutSFB layout_SFB = swap_ab ?
ScaleConfig::tile_atom_to_shape_SFB(make_shape(n, m, k, 1)) :
ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1));
auto a_ptr = static_cast<ElementAB const*>(a.data_ptr());
auto b_ptr = static_cast<ElementAB const*>(b.data_ptr());
auto a_scales_ptr = static_cast<ElementBlockScale const*>(a_scales.data_ptr());
auto a_scales_ptr = static_cast<ElementBlockScale const*>(a_scales_data);
auto b_scales_ptr = static_cast<ElementBlockScale const*>(b_scales.data_ptr());
typename GemmKernel::MainloopArguments mainloop_args{};
@@ -212,7 +235,9 @@ void cutlass_gemm_blockwise_sm100_fp8_dispatch(torch::stable::Tensor& out,
constexpr int TILE_K = 128;
// TODO: better heuristics
bool swap_ab = (m < 16) || (m % 4 != 0);
// swapAB only wins for small M; misaligned M >= 65 is faster on the
// non-swapAB path with padded activation scales (measured on GB300).
bool swap_ab = (m <= 64);
bool use_tma_epilogue = (m * n) % 4 == 0;
if (!swap_ab) {
constexpr int TILE_N = 128;
+7 -2
View File
@@ -162,11 +162,16 @@ def test_w8a8_block_fp8_matmul(M, N, K, block_size, out_dtype, seed):
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUTLASS only supported on CUDA platform."
)
@pytest.mark.parametrize(
# 65/66/67 cover all M%4 residue classes above the SM100 swapAB
# threshold (m <= 64); 1026 crosses multiple 128-row SF atoms.
"M",
[32, 65, 66, 67, 1026],
)
@torch.inference_mode()
def test_w8a8_block_fp8_cutlass_matmul():
def test_w8a8_block_fp8_cutlass_matmul(M):
# Test simple case where weight.shape % 128 != 0,
# like in DSV3 kv_a_proj_with_mqa
M = 32
N = 576
K = 7168
block_size = [128, 128]