From 514c7314a06be9e43d00d3f126a5aaa54211274f Mon Sep 17 00:00:00 2001 From: "Roberto L. Castro" <38211239+LopezCastroRoberto@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:49:07 +0200 Subject: [PATCH] [Perf][Kernel] Initialize NVFP4 padding in quant kernel (#53568) Signed-off-by: LopezCastroRoberto Co-authored-by: Codex Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../quantization/fp4/nvfp4_quant_kernels.cu | 67 +++++++++++++------ .../kernels/quantization/test_nvfp4_quant.py | 41 +++++++++++- vllm/_custom_ops.py | 10 +-- 3 files changed, 91 insertions(+), 27 deletions(-) diff --git a/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu b/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu index 97d04f96bd..7ec418ee38 100644 --- a/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu +++ b/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu @@ -36,7 +36,7 @@ namespace vllm { // Use UE4M3 by default. -template +template __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) cvt_fp16_to_fp4(int32_t numRows, int32_t numCols, int32_t outputCols, int32_t num_padded_cols, Type const* __restrict__ in, @@ -55,7 +55,7 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) #endif // Precompute SF layout parameter (constant for entire kernel). - int32_t const numKTiles = (outputCols + 63) / 64; + int32_t const numKTiles = ((PADDED_OUTPUT ? outputCols : numCols) + 63) / 64; int sf_m = round_up(numRows, 128); int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x; @@ -74,16 +74,15 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) int64_t inOffset = rowIdx * (numCols / CVT_FP4_ELTS_PER_THREAD) + colIdx; // If we are outside valid columns, feed zeros - bool valid_input = (rowIdx < numRows) && (elem_idx < numCols); - bool valid_output = (rowIdx < numRows) && (elem_idx < outputCols); + bool valid = (rowIdx < numRows) && (elem_idx < numCols); if constexpr (CVT_FP4_PACK16) { ld256_cg_or_zero(reinterpret_cast(in_vec), &reinterpret_cast(in)[inOffset * 8], - valid_input); + valid); } else { ld128_cg_or_zero(reinterpret_cast(in_vec), &reinterpret_cast(in)[inOffset * 4], - valid_input); + valid); } auto sf_out = @@ -91,20 +90,42 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) CVT_FP4_NUM_THREADS_PER_SF>( rowIdx, colIdx, numKTiles, SFout); - auto out_val = - cvt_warp_fp16_to_fp4( - in_vec, global_scale, sf_out); + fp4_packed_t out_val{}; + if (valid) { + out_val = + cvt_warp_fp16_to_fp4( + in_vec, global_scale, sf_out); + } else if (sf_out != nullptr) { + // The swizzled scale tensor is allocated with torch.empty. Explicitly + // initialize padded rows and columns in this kernel so downstream + // NVFP4 GEMMs never observe stale scale factors. + *sf_out = 0; + } - if (valid_output) { + if constexpr (PADDED_OUTPUT) { + bool valid_output = (rowIdx < numRows) && (elem_idx < outputCols); + if (valid_output) { + if constexpr (CVT_FP4_PACK16) { + int64_t outOffset = rowIdx * (outputCols / 8) + colIdx * 2; + uint64_t packed64 = + (uint64_t(out_val.hi) << 32) | uint64_t(out_val.lo); + reinterpret_cast(out)[outOffset >> 1] = packed64; + } else { + int64_t outOffset = + rowIdx * (outputCols / CVT_FP4_ELTS_PER_THREAD) + colIdx; + reinterpret_cast(out)[outOffset] = out_val; + } + } + } else if (valid) { + // #32520 fast path: input and output widths are identical, so avoid + // the padded-output predicate and use the original address arithmetic. if constexpr (CVT_FP4_PACK16) { - int64_t outOffset = rowIdx * (outputCols / 8) + colIdx * 2; + int64_t outOffset = rowIdx * (numCols / 8) + colIdx * 2; uint64_t packed64 = (uint64_t(out_val.hi) << 32) | uint64_t(out_val.lo); reinterpret_cast(out)[outOffset >> 1] = packed64; } else { - int64_t outOffset = - rowIdx * (outputCols / CVT_FP4_ELTS_PER_THREAD) + colIdx; - out[outOffset] = out_val; + reinterpret_cast(out)[inOffset] = out_val; } } } @@ -244,11 +265,19 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output, attrs[0].val.programmaticStreamSerializationAllowed = 1; config.numAttrs = 1; config.attrs = attrs; - cudaLaunchKernelEx(&config, vllm::cvt_fp16_to_fp4, - m, n, output_n, num_padded_cols, input_ptr, - input_sf_ptr, - reinterpret_cast(output_ptr), - reinterpret_cast(sf_out)); + if (output_n == n) { + cudaLaunchKernelEx( + &config, vllm::cvt_fp16_to_fp4, m, n, + output_n, num_padded_cols, input_ptr, input_sf_ptr, + reinterpret_cast(output_ptr), + reinterpret_cast(sf_out)); + } else { + cudaLaunchKernelEx( + &config, vllm::cvt_fp16_to_fp4, m, n, + output_n, num_padded_cols, input_ptr, input_sf_ptr, + reinterpret_cast(output_ptr), + reinterpret_cast(sf_out)); + } }); } else { int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD; diff --git a/tests/kernels/quantization/test_nvfp4_quant.py b/tests/kernels/quantization/test_nvfp4_quant.py index f4eaf7187e..120dc2774a 100644 --- a/tests/kernels/quantization/test_nvfp4_quant.py +++ b/tests/kernels/quantization/test_nvfp4_quant.py @@ -119,7 +119,7 @@ def ref_nvfp4_quant(x, global_scale): return cast_to_fp4(clipped_x), scale.squeeze(-1) -def recover_swizzled_scales(scale, m, n): +def recover_swizzled_scales(scale, m, n, include_padding=False): round_up = lambda x, y: (x + y - 1) // y * y rounded_m = round_up(m, 128) scale_n = n // BLOCK_SIZE @@ -128,6 +128,8 @@ def recover_swizzled_scales(scale, m, n): tmp = torch.reshape(scale, (1, rounded_m // 128, rounded_n // 4, 32, 4, 4)) tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5)) result = torch.reshape(tmp, (rounded_m, rounded_n)).to(torch.float32) + if include_padding: + return result return result[:m, :scale_n] @@ -264,6 +266,43 @@ def test_quantize_to_fp4_with_padded_output( assert torch.count_nonzero(scale_ans[:, n // BLOCK_SIZE :]) == 0 +@pytest.mark.parametrize( + ("shape", "padded_n"), + [((90, 80), None), ((90, 48), 64)], +) +@torch.inference_mode() +def test_swizzled_scale_padding_initialized_in_kernel( + shape: tuple[int, int], padded_n: int | None +) -> None: + """The out variant must overwrite dirty scale padding without a memset.""" + torch.set_default_device("cuda:0") + m, n = shape + physical_n = padded_n if padded_n is not None else n + rounded_m = round_up(m, 128) + rounded_scale_n = round_up(physical_n // BLOCK_SIZE, 4) + + x = torch.randn((m, n), dtype=torch.float16) + global_scale = torch.tensor(1.0, dtype=torch.float32) + out = torch.full((m, physical_n // 2), 0xFF, dtype=torch.uint8) + out_scale = torch.full( + (rounded_m, rounded_scale_n // 4), 0x7F7F7F7F, dtype=torch.int32 + ) + + torch.ops._C.scaled_fp4_quant.out( + x, + global_scale, + True, + output=out, + output_scale=out_scale, + ) + + full_scale = recover_swizzled_scales( + out_scale.view(torch.float8_e4m3fn), m, physical_n, include_padding=True + ) + assert torch.count_nonzero(full_scale[m:, :]) == 0 + assert torch.count_nonzero(full_scale[:m, n // BLOCK_SIZE :]) == 0 + + @pytest.mark.parametrize("pad_shape", PAD_SHAPES) @torch.inference_mode() def test_quantize_to_fp4_padded(pad_shape: tuple[int, int]) -> None: diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index b01489a447..8c8d28b04b 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -57,13 +57,9 @@ def create_fp4_scale_tensor( rounded_m = round_up(m, 128) scale_n = n // block_size rounded_n = round_up(scale_n, 4) - # Must be zero-initialized: the swizzled scale buffer is padded to - # (round_up(m, 128), round_up(scale_n, 4) // 4) but the NVFP4 quant - # kernel does not write every padded element that the downstream - # NVFP4 GEMM reads. torch.empty leaves those padded scale factors - # uninitialized, which corrupts dequantization and causes a severe - # Blackwell NVFP4 decode throughput/output-length regression. - return torch.zeros( + # The NVFP4 quant kernel explicitly zeroes every padded scale entry, + # so no separate zero-initialization kernel is required here. + return torch.empty( (rounded_m, rounded_n // 4), device=device, dtype=torch.int32 ) else: