C3x SM100 FP8 blockwise - Pad activation scales to multiple of 4 (#57377)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -367,6 +367,7 @@ steps:
|
||||
- csrc/quantization/fp4/
|
||||
- csrc/libtorch_stable/nvfp4_kv_cache_kernels.cu
|
||||
- csrc/libtorch_stable/quantization/fp4/
|
||||
- csrc/libtorch_stable/quantization/w8a8/cutlass/
|
||||
- csrc/attention/mla/
|
||||
- csrc/quantization/cutlass_w8a8/moe/
|
||||
- vllm/model_executor/layers/fusion/
|
||||
@@ -408,6 +409,7 @@ steps:
|
||||
- tests/kernels/test_ll_bf16_gemm.py
|
||||
- tests/kernels/test_top_k_per_row.py
|
||||
- tests/kernels/quantization/test_silu_mul_nvfp4_quant.py
|
||||
- tests/kernels/quantization/test_block_fp8.py
|
||||
- tests/models/quantization/test_nvfp4.py
|
||||
commands:
|
||||
- nvidia-smi
|
||||
@@ -442,6 +444,7 @@ steps:
|
||||
pytest -v -s tests/kernels/attention/test_flashinfer.py -k 'not num_heads2'
|
||||
pytest -v -s tests/kernels/attention/test_cutlass_mla_decode.py
|
||||
pytest -v -s tests/kernels/quantization/test_cutlass_scaled_mm.py -k 'fp8'
|
||||
pytest -v -s tests/kernels/quantization/test_block_fp8.py::test_w8a8_block_fp8_cutlass_matmul
|
||||
pytest -v -s tests/kernels/quantization/test_nvfp4_quant.py
|
||||
pytest -v -s tests/kernels/quantization/test_nvfp4_scaled_mm.py
|
||||
pytest -v -s tests/kernels/quantization/test_mxfp4_qutlass.py
|
||||
|
||||
+29
-4
@@ -152,6 +152,29 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te
|
||||
|
||||
int32_t m = a.size(0), n = b.size(1), k = a.size(1);
|
||||
|
||||
// The SM100 blockwise SF copy requires the SFA M extent to be a multiple
|
||||
// of 4 (4-row copy atom). For misaligned M on the non-swapAB path, pad only
|
||||
// the per-token activation scales (column-major (m, k/128), 1/64 the size
|
||||
// of the bf16 activation) and build layout_SFA from the padded extent.
|
||||
int32_t m_sf = m;
|
||||
std::optional<torch::stable::Tensor> a_scales_pad;
|
||||
void const* a_scales_data = a_scales.data_ptr();
|
||||
if constexpr (!swap_ab) {
|
||||
if (m % 4 != 0) {
|
||||
m_sf = (m + 3) & ~3;
|
||||
int64_t num_groups = a_scales.numel() / m;
|
||||
a_scales_pad = torch::stable::new_empty(
|
||||
a_scales, {num_groups * m_sf},
|
||||
torch::headeronly::ScalarType::Float);
|
||||
cudaMemcpy2DAsync(a_scales_pad->data_ptr(), m_sf * sizeof(float),
|
||||
a_scales.data_ptr(), m * sizeof(float),
|
||||
m * sizeof(float), num_groups,
|
||||
cudaMemcpyDeviceToDevice,
|
||||
get_current_cuda_stream(a.get_device()));
|
||||
a_scales_data = a_scales_pad->data_ptr();
|
||||
}
|
||||
}
|
||||
|
||||
StrideA a_stride;
|
||||
StrideB b_stride;
|
||||
StrideC c_stride;
|
||||
@@ -162,16 +185,16 @@ void cutlass_gemm_caller_blockwise(torch::stable::Tensor& out, torch::stable::Te
|
||||
c_stride =
|
||||
cutlass::make_cute_packed_stride(StrideC{}, swap_ab ? cute::make_shape(n, m, 1) : cute::make_shape(m, n, 1));
|
||||
|
||||
LayoutSFA layout_SFA = swap_ab ?
|
||||
LayoutSFA layout_SFA = swap_ab ?
|
||||
ScaleConfig::tile_atom_to_shape_SFA(make_shape(n, m, k, 1)) :
|
||||
ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1));
|
||||
ScaleConfig::tile_atom_to_shape_SFA(make_shape(m_sf, n, k, 1));
|
||||
LayoutSFB layout_SFB = swap_ab ?
|
||||
ScaleConfig::tile_atom_to_shape_SFB(make_shape(n, m, k, 1)) :
|
||||
ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1));
|
||||
|
||||
auto a_ptr = static_cast<ElementAB const*>(a.data_ptr());
|
||||
auto b_ptr = static_cast<ElementAB const*>(b.data_ptr());
|
||||
auto a_scales_ptr = static_cast<ElementBlockScale const*>(a_scales.data_ptr());
|
||||
auto a_scales_ptr = static_cast<ElementBlockScale const*>(a_scales_data);
|
||||
auto b_scales_ptr = static_cast<ElementBlockScale const*>(b_scales.data_ptr());
|
||||
|
||||
typename GemmKernel::MainloopArguments mainloop_args{};
|
||||
@@ -212,7 +235,9 @@ void cutlass_gemm_blockwise_sm100_fp8_dispatch(torch::stable::Tensor& out,
|
||||
|
||||
constexpr int TILE_K = 128;
|
||||
// TODO: better heuristics
|
||||
bool swap_ab = (m < 16) || (m % 4 != 0);
|
||||
// swapAB only wins for small M; misaligned M >= 65 is faster on the
|
||||
// non-swapAB path with padded activation scales (measured on GB300).
|
||||
bool swap_ab = (m <= 64);
|
||||
bool use_tma_epilogue = (m * n) % 4 == 0;
|
||||
if (!swap_ab) {
|
||||
constexpr int TILE_N = 128;
|
||||
|
||||
@@ -162,11 +162,16 @@ def test_w8a8_block_fp8_matmul(M, N, K, block_size, out_dtype, seed):
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="CUTLASS only supported on CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
# 65/66/67 cover all M%4 residue classes above the SM100 swapAB
|
||||
# threshold (m <= 64); 1026 crosses multiple 128-row SF atoms.
|
||||
"M",
|
||||
[32, 65, 66, 67, 1026],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_cutlass_matmul():
|
||||
def test_w8a8_block_fp8_cutlass_matmul(M):
|
||||
# Test simple case where weight.shape % 128 != 0,
|
||||
# like in DSV3 kv_a_proj_with_mqa
|
||||
M = 32
|
||||
N = 576
|
||||
K = 7168
|
||||
block_size = [128, 128]
|
||||
|
||||
Reference in New Issue
Block a user