diff --git a/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu b/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu index beb1287c83..e1d101f9d8 100644 --- a/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu +++ b/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu @@ -38,8 +38,8 @@ namespace vllm { // Use UE4M3 by default. template __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) - cvt_fp16_to_fp4(int32_t numRows, int32_t numCols, int32_t num_padded_cols, - Type const* __restrict__ in, + cvt_fp16_to_fp4(int32_t numRows, int32_t numCols, int32_t outputCols, + int32_t num_padded_cols, Type const* __restrict__ in, float const* __restrict__ SFScale, uint32_t* __restrict__ out, uint32_t* __restrict__ SFout) { using PackedVec = vllm::PackedVec; @@ -50,7 +50,7 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) "Vec size is not matched."); // Precompute SF layout parameter (constant for entire kernel). - int32_t const numKTiles = (numCols + 63) / 64; + int32_t const numKTiles = (outputCols + 63) / 64; int sf_m = round_up(numRows, 128); int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x; @@ -68,16 +68,17 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) PackedVec in_vec; int64_t inOffset = rowIdx * (numCols / CVT_FP4_ELTS_PER_THREAD) + colIdx; - // If we are outside valid rows OR outside valid columns -> Use Zeros - bool valid = (rowIdx < numRows) && (elem_idx < numCols); + // If we are outside valid columns, feed zeros + bool valid_input = (rowIdx < numRows) && (elem_idx < numCols); + bool valid_output = (rowIdx < numRows) && (elem_idx < outputCols); if constexpr (CVT_FP4_PACK16) { ld256_cg_or_zero(reinterpret_cast(in_vec), &reinterpret_cast(in)[inOffset * 8], - valid); + valid_input); } else { ld128_cg_or_zero(reinterpret_cast(in_vec), &reinterpret_cast(in)[inOffset * 4], - valid); + valid_input); } auto sf_out = @@ -89,16 +90,16 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) cvt_warp_fp16_to_fp4( in_vec, global_scale, sf_out); - // We do NOT write output for padding because the 'out' tensor is not - // padded. - if (valid) { + if (valid_output) { if constexpr (CVT_FP4_PACK16) { - int64_t outOffset = rowIdx * (numCols / 8) + colIdx * 2; + int64_t outOffset = rowIdx * (outputCols / 8) + colIdx * 2; uint64_t packed64 = (uint64_t(out_val.hi) << 32) | uint64_t(out_val.lo); reinterpret_cast(out)[outOffset >> 1] = packed64; } else { - out[inOffset] = out_val; + int64_t outOffset = + rowIdx * (outputCols / CVT_FP4_ELTS_PER_THREAD) + colIdx; + out[outOffset] = out_val; } } } @@ -109,7 +110,8 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) template __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) cvt_fp16_to_fp4_sf_major(int32_t numRows, int32_t numCols, - int32_t sf_n_unpadded, int32_t num_packed_cols, + int32_t outputCols, int32_t sf_n_unpadded, + int32_t num_packed_cols, Type const* __restrict__ in, float const* __restrict__ SFScale, uint32_t* __restrict__ out, @@ -136,7 +138,7 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) PackedVec in_vec; int64_t inOffset = rowIdx * (numCols / CVT_FP4_ELTS_PER_THREAD) + colIdx; - // If we are outside valid rows OR outside valid columns -> Use Zeros + // If we are outside valid columns, feed zeros bool valid = (rowIdx < numRows) && (elem_idx < numCols); if constexpr (CVT_FP4_PACK16) { ld256_cg_or_zero(reinterpret_cast(in_vec), @@ -155,16 +157,16 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) cvt_warp_fp16_to_fp4( in_vec, global_scale, sf_out); - // We do NOT write output for padding because the 'out' tensor is not - // padded. - if (valid) { + if (rowIdx < numRows) { if constexpr (CVT_FP4_PACK16) { - int64_t outOffset = rowIdx * (numCols / 8) + colIdx * 2; + int64_t outOffset = rowIdx * (outputCols / 8) + colIdx * 2; uint64_t packed64 = (uint64_t(out_val.hi) << 32) | uint64_t(out_val.lo); reinterpret_cast(out)[outOffset >> 1] = packed64; } else { - out[inOffset] = out_val; + int64_t outOffset = + rowIdx * (outputCols / CVT_FP4_ELTS_PER_THREAD) + colIdx; + out[outOffset] = out_val; } } } @@ -180,8 +182,11 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output, bool is_sf_swizzled_layout) { int32_t m = input.size(0); int32_t n = input.size(1); + int32_t output_n = output.size(1) * 2; STD_TORCH_CHECK(n % 16 == 0, "The N dimension must be multiple of 16."); + STD_TORCH_CHECK(output_n % 16 == 0, + "The output tensor width must be a multiple of 16."); STD_TORCH_CHECK( input.scalar_type() == torch::headeronly::ScalarType::Half || input.scalar_type() == torch::headeronly::ScalarType::BFloat16, @@ -197,7 +202,7 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output, input.get_device_index()); auto stream = get_current_cuda_stream(input.get_device_index()); - int sf_n_unpadded = int(n / CVT_FP4_SF_VEC_SIZE); + int output_sf_n_unpadded = int(output_n / CVT_FP4_SF_VEC_SIZE); // Grid, Block size. Each thread converts 8 values. dim3 block(std::min(int(n / ELTS_PER_THREAD), 512)); @@ -205,7 +210,7 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output, vllm_runtime_blocks_per_sm(static_cast(block.x)); if (is_sf_swizzled_layout) { - int sf_n_int = int(vllm::round_up(sf_n_unpadded, 4) / 4); + int sf_n_int = int(vllm::round_up(output_sf_n_unpadded, 4) / 4); int32_t num_padded_cols = sf_n_int * 4 * CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD; @@ -220,12 +225,12 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output, using cuda_type = vllm::CUDATypeConverter::Type; auto input_ptr = static_cast(input.data_ptr()); vllm::cvt_fp16_to_fp4<<>>( - m, n, num_padded_cols, input_ptr, input_sf_ptr, + m, n, output_n, num_padded_cols, input_ptr, input_sf_ptr, reinterpret_cast(output_ptr), reinterpret_cast(sf_out)); }); } else { - int num_packed_cols = n / CVT_FP4_ELTS_PER_THREAD; + int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD; int grid_y = vllm::div_round_up(num_packed_cols, static_cast(block.x)); int grid_x = std::min( m, std::max(1, (multiProcessorCount * numBlocksPerSM) / grid_y)); @@ -237,7 +242,8 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output, auto input_ptr = static_cast(input.data_ptr()); vllm::cvt_fp16_to_fp4_sf_major <<>>( - m, n, sf_n_unpadded, num_packed_cols, input_ptr, input_sf_ptr, + m, n, output_n, output_sf_n_unpadded, num_packed_cols, + input_ptr, input_sf_ptr, reinterpret_cast(output_ptr), reinterpret_cast(sf_out)); }); diff --git a/tests/kernels/quantization/test_nvfp4_quant.py b/tests/kernels/quantization/test_nvfp4_quant.py index e2db597588..f4eaf7187e 100644 --- a/tests/kernels/quantization/test_nvfp4_quant.py +++ b/tests/kernels/quantization/test_nvfp4_quant.py @@ -34,6 +34,7 @@ PAD_SHAPES = [ (64, 7152), (32, 14336), ] +PADDED_OUTPUT_SHAPES = [(128, 48), (128, 80), (150, 48), (150, 80), (64, 7152)] SEEDS = [42] CUDA_DEVICES = ["cuda:0"] @@ -130,6 +131,10 @@ def recover_swizzled_scales(scale, m, n): return result[:m, :scale_n] +def round_up(x: int, y: int) -> int: + return (x + y - 1) // y * y + + @pytest.mark.parametrize("dtype", DTYPES) @pytest.mark.parametrize("shape", SHAPES) @pytest.mark.parametrize("seed", SEEDS) @@ -205,6 +210,60 @@ def test_python_util_matches_cpp_allocation( ) +@pytest.mark.parametrize("shape", PADDED_OUTPUT_SHAPES) +@pytest.mark.parametrize("is_sf_swizzled_layout", [True, False]) +@torch.inference_mode() +def test_quantize_to_fp4_with_padded_output( + shape: tuple[int, int], + is_sf_swizzled_layout: bool, +) -> None: + from vllm._custom_ops import create_fp4_output_tensors + + dtype = torch.float16 + set_random_seed(42) + torch.set_default_device("cuda:0") + + m, n = shape + padded_n = round_up(n, 32) + assert padded_n > n + + x = torch.randn((m, n), dtype=dtype) + tensor_amax = torch.abs(x).max().to(torch.float32) + global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax + out_ref, scale_ref = ref_nvfp4_quant(x, global_scale) + + out, out_scale = ops.scaled_fp4_quant( + x, + global_scale, + is_sf_swizzled_layout=is_sf_swizzled_layout, + padded_n=padded_n, + ) + py_out, py_scale = create_fp4_output_tensors( + m, + n, + torch.device("cuda:0"), + is_sf_swizzled_layout, + padded_n=padded_n, + ) + + assert out.shape == (m, padded_n // 2) + assert out.shape == py_out.shape + assert out_scale.shape == py_scale.view(torch.float8_e4m3fn).shape + + out_ans = cast_from_fp4(out[:, : n // 2], m, n) + torch.testing.assert_close(out_ans, out_ref) + assert torch.count_nonzero(out[:, n // 2 :]) == 0 + + if is_sf_swizzled_layout: + scale_ans = recover_swizzled_scales(out_scale, m, padded_n) + torch.testing.assert_close(scale_ans[:, : n // BLOCK_SIZE], scale_ref) + assert torch.count_nonzero(scale_ans[:, n // BLOCK_SIZE :]) == 0 + else: + scale_ans = out_scale.to(torch.float32) + torch.testing.assert_close(scale_ans[:, : n // BLOCK_SIZE], scale_ref) + assert torch.count_nonzero(scale_ans[:, n // BLOCK_SIZE :]) == 0 + + @pytest.mark.parametrize("pad_shape", PAD_SHAPES) @torch.inference_mode() def test_quantize_to_fp4_padded(pad_shape: tuple[int, int]) -> None: diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 6299cf15d6..cb4bd28f26 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -69,15 +69,20 @@ def create_fp4_output_tensors( n: int, device: torch.device, is_sf_swizzled_layout: bool, + padded_n: int | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: """ Allocate both output tensors for scaled_fp4_quant: (quantized_output, output_scale). - Must match the C++ scaled_fp4_quant_func allocation exactly. + Must match the C++ scaled_fp4_quant_func allocation exactly when + ``padded_n`` is ``None``. When ``padded_n`` is provided, allocate a larger + packed-FP4 output/scale buffer so the quantization kernel can write + CUTLASS-compatible K padding directly """ - output = torch.empty((m, n // 2), device=device, dtype=torch.uint8) - output_scale = create_fp4_scale_tensor(m, n, device, is_sf_swizzled_layout) + physical_n = padded_n if padded_n is not None else n + output = torch.empty((m, physical_n // 2), device=device, dtype=torch.uint8) + output_scale = create_fp4_scale_tensor(m, physical_n, device, is_sf_swizzled_layout) return output, output_scale @@ -1597,6 +1602,7 @@ def scaled_fp4_quant( input_global_scale: torch.Tensor, is_sf_swizzled_layout: bool = True, backend: str = "none", + padded_n: int | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: """ Quantize input tensor to FP4 and return quantized tensor and scale. @@ -1611,6 +1617,8 @@ def scaled_fp4_quant( input: The input tensor to be quantized to FP4 input_global_scale: A scalar scaling factor for the entire tensor. use_8x4_sf_layout: Whether to use the 8x4 or 128x4 layout for the scaling + padded_n: Optional padded K dimension. When provided, the quantized + output and scale tensors are allocated for ``padded_n`` Returns: tuple[torch.Tensor, torch.Tensor]: The output tensor in FP4 but every @@ -1628,9 +1636,16 @@ def scaled_fp4_quant( assert input.dtype in (torch.float16, torch.bfloat16), ( f"input.dtype needs to be fp16 or bf16 but got {input.dtype}." ) + if padded_n is not None: + assert padded_n >= n, f"padded_n must be >= n, got padded_n={padded_n}, n={n}." + assert padded_n % block_size == 0, ( + f"padded_n has to be a multiple of {block_size}, but got {padded_n}." + ) use_8x4_sf_layout = True if "trtllm" in backend and m <= 32 else False # noqa: SIM210 - + if use_8x4_sf_layout and padded_n is not None and padded_n != n: + # TODO: support this case + raise ValueError("padded_n is not supported with TRTLLM 8x4 scale layout.") if use_8x4_sf_layout: output, output_scale = flashinfer_quant_nvfp4_8x4_sf_layout( input, input_global_scale @@ -1638,7 +1653,11 @@ def scaled_fp4_quant( else: # Pre-allocate and call .out variant (same behavior as old in-place API) output, output_scale = create_fp4_output_tensors( - m, n, input.device, is_sf_swizzled_layout + m, + n, + input.device, + is_sf_swizzled_layout, + padded_n=padded_n, ) torch.ops._C.scaled_fp4_quant.out( input, diff --git a/vllm/model_executor/kernels/linear/nvfp4/cutlass.py b/vllm/model_executor/kernels/linear/nvfp4/cutlass.py index 0d0663dca1..106e92d912 100644 --- a/vllm/model_executor/kernels/linear/nvfp4/cutlass.py +++ b/vllm/model_executor/kernels/linear/nvfp4/cutlass.py @@ -9,7 +9,6 @@ from vllm._custom_ops import ( ) from vllm.model_executor.layers.quantization.utils.nvfp4_utils import ( cutlass_fp4_supported, - pad_nvfp4_activation_for_cutlass, pad_nvfp4_weight_for_cutlass, slice_nvfp4_output, swizzle_blockscale, @@ -52,16 +51,14 @@ class CutlassNvFp4LinearKernel(NvFp4LinearKernel): output_size = layer.output_size_per_partition output_dtype = x.dtype output_shape = [*x.shape[:-1], output_size] + weights_padding_bytes = getattr(layer, "weights_padding_cols", 0) x_fp4, x_blockscale = scaled_fp4_quant( x, layer.input_global_scale_inv, is_sf_swizzled_layout=True, backend="cutlass", - ) - - x_fp4 = pad_nvfp4_activation_for_cutlass( - x_fp4, getattr(layer, "weights_padding_cols", 0) + padded_n=x.shape[-1] + weights_padding_bytes * 2, ) out = cutlass_scaled_fp4_mm( diff --git a/vllm/model_executor/kernels/linear/nvfp4/flashinfer.py b/vllm/model_executor/kernels/linear/nvfp4/flashinfer.py index 399bc3dd27..fd8f4de101 100644 --- a/vllm/model_executor/kernels/linear/nvfp4/flashinfer.py +++ b/vllm/model_executor/kernels/linear/nvfp4/flashinfer.py @@ -5,7 +5,6 @@ import torch from vllm._custom_ops import scaled_fp4_quant from vllm.model_executor.layers.quantization.utils.nvfp4_utils import ( - pad_nvfp4_activation_for_cutlass, pad_nvfp4_weight_for_cutlass, slice_nvfp4_output, swizzle_blockscale, @@ -58,16 +57,14 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel): output_size = layer.output_size_per_partition output_dtype = x.dtype output_shape = [*x.shape[:-1], output_size] + weights_padding_bytes = getattr(layer, "weights_padding_cols", 0) x_fp4, x_blockscale = scaled_fp4_quant( x, layer.input_global_scale_inv, is_sf_swizzled_layout=True, backend="flashinfer-cutlass", - ) - - x_fp4 = pad_nvfp4_activation_for_cutlass( - x_fp4, getattr(layer, "weights_padding_cols", 0) + padded_n=x.shape[-1] + weights_padding_bytes * 2, ) out = flashinfer_scaled_fp4_mm( @@ -189,16 +186,14 @@ class FlashInferCudnnNvFp4LinearKernel(NvFp4LinearKernel): output_size = layer.output_size_per_partition output_dtype = x.dtype output_shape = [*x.shape[:-1], output_size] + weights_padding_bytes = getattr(layer, "weights_padding_cols", 0) x_fp4, x_blockscale = scaled_fp4_quant( x, layer.input_global_scale_inv, is_sf_swizzled_layout=True, backend="flashinfer-cudnn", - ) - - x_fp4 = pad_nvfp4_activation_for_cutlass( - x_fp4, getattr(layer, "weights_padding_cols", 0) + padded_n=x.shape[-1] + weights_padding_bytes * 2, ) out = flashinfer_scaled_fp4_mm(