[Perf] Padded nvfp4 quant kernel to remove additional copy, 2.4%~5.7% e2e performance improvement (#42774)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -38,8 +38,8 @@ namespace vllm {
|
||||
// Use UE4M3 by default.
|
||||
template <class Type, bool UE8M0_SF = false>
|
||||
__global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
cvt_fp16_to_fp4(int32_t numRows, int32_t numCols, int32_t num_padded_cols,
|
||||
Type const* __restrict__ in,
|
||||
cvt_fp16_to_fp4(int32_t numRows, int32_t numCols, int32_t outputCols,
|
||||
int32_t num_padded_cols, Type const* __restrict__ in,
|
||||
float const* __restrict__ SFScale,
|
||||
uint32_t* __restrict__ out, uint32_t* __restrict__ SFout) {
|
||||
using PackedVec = vllm::PackedVec<Type, CVT_FP4_PACK16>;
|
||||
@@ -50,7 +50,7 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
"Vec size is not matched.");
|
||||
|
||||
// Precompute SF layout parameter (constant for entire kernel).
|
||||
int32_t const numKTiles = (numCols + 63) / 64;
|
||||
int32_t const numKTiles = (outputCols + 63) / 64;
|
||||
|
||||
int sf_m = round_up<int>(numRows, 128);
|
||||
int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x;
|
||||
@@ -68,16 +68,17 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
PackedVec in_vec;
|
||||
int64_t inOffset = rowIdx * (numCols / CVT_FP4_ELTS_PER_THREAD) + colIdx;
|
||||
|
||||
// If we are outside valid rows OR outside valid columns -> Use Zeros
|
||||
bool valid = (rowIdx < numRows) && (elem_idx < numCols);
|
||||
// If we are outside valid columns, feed zeros
|
||||
bool valid_input = (rowIdx < numRows) && (elem_idx < numCols);
|
||||
bool valid_output = (rowIdx < numRows) && (elem_idx < outputCols);
|
||||
if constexpr (CVT_FP4_PACK16) {
|
||||
ld256_cg_or_zero(reinterpret_cast<u32x8_t&>(in_vec),
|
||||
&reinterpret_cast<const uint32_t*>(in)[inOffset * 8],
|
||||
valid);
|
||||
valid_input);
|
||||
} else {
|
||||
ld128_cg_or_zero(reinterpret_cast<uint4&>(in_vec),
|
||||
&reinterpret_cast<const uint32_t*>(in)[inOffset * 4],
|
||||
valid);
|
||||
valid_input);
|
||||
}
|
||||
|
||||
auto sf_out =
|
||||
@@ -89,16 +90,16 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
cvt_warp_fp16_to_fp4<Type, CVT_FP4_NUM_THREADS_PER_SF, UE8M0_SF>(
|
||||
in_vec, global_scale, sf_out);
|
||||
|
||||
// We do NOT write output for padding because the 'out' tensor is not
|
||||
// padded.
|
||||
if (valid) {
|
||||
if (valid_output) {
|
||||
if constexpr (CVT_FP4_PACK16) {
|
||||
int64_t outOffset = rowIdx * (numCols / 8) + colIdx * 2;
|
||||
int64_t outOffset = rowIdx * (outputCols / 8) + colIdx * 2;
|
||||
uint64_t packed64 =
|
||||
(uint64_t(out_val.hi) << 32) | uint64_t(out_val.lo);
|
||||
reinterpret_cast<uint64_t*>(out)[outOffset >> 1] = packed64;
|
||||
} else {
|
||||
out[inOffset] = out_val;
|
||||
int64_t outOffset =
|
||||
rowIdx * (outputCols / CVT_FP4_ELTS_PER_THREAD) + colIdx;
|
||||
out[outOffset] = out_val;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -109,7 +110,8 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
template <class Type, bool UE8M0_SF = false>
|
||||
__global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
cvt_fp16_to_fp4_sf_major(int32_t numRows, int32_t numCols,
|
||||
int32_t sf_n_unpadded, int32_t num_packed_cols,
|
||||
int32_t outputCols, int32_t sf_n_unpadded,
|
||||
int32_t num_packed_cols,
|
||||
Type const* __restrict__ in,
|
||||
float const* __restrict__ SFScale,
|
||||
uint32_t* __restrict__ out,
|
||||
@@ -136,7 +138,7 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
PackedVec in_vec;
|
||||
int64_t inOffset = rowIdx * (numCols / CVT_FP4_ELTS_PER_THREAD) + colIdx;
|
||||
|
||||
// If we are outside valid rows OR outside valid columns -> Use Zeros
|
||||
// If we are outside valid columns, feed zeros
|
||||
bool valid = (rowIdx < numRows) && (elem_idx < numCols);
|
||||
if constexpr (CVT_FP4_PACK16) {
|
||||
ld256_cg_or_zero(reinterpret_cast<u32x8_t&>(in_vec),
|
||||
@@ -155,16 +157,16 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
cvt_warp_fp16_to_fp4<Type, CVT_FP4_NUM_THREADS_PER_SF, UE8M0_SF>(
|
||||
in_vec, global_scale, sf_out);
|
||||
|
||||
// We do NOT write output for padding because the 'out' tensor is not
|
||||
// padded.
|
||||
if (valid) {
|
||||
if (rowIdx < numRows) {
|
||||
if constexpr (CVT_FP4_PACK16) {
|
||||
int64_t outOffset = rowIdx * (numCols / 8) + colIdx * 2;
|
||||
int64_t outOffset = rowIdx * (outputCols / 8) + colIdx * 2;
|
||||
uint64_t packed64 =
|
||||
(uint64_t(out_val.hi) << 32) | uint64_t(out_val.lo);
|
||||
reinterpret_cast<uint64_t*>(out)[outOffset >> 1] = packed64;
|
||||
} else {
|
||||
out[inOffset] = out_val;
|
||||
int64_t outOffset =
|
||||
rowIdx * (outputCols / CVT_FP4_ELTS_PER_THREAD) + colIdx;
|
||||
out[outOffset] = out_val;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -180,8 +182,11 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
|
||||
bool is_sf_swizzled_layout) {
|
||||
int32_t m = input.size(0);
|
||||
int32_t n = input.size(1);
|
||||
int32_t output_n = output.size(1) * 2;
|
||||
|
||||
STD_TORCH_CHECK(n % 16 == 0, "The N dimension must be multiple of 16.");
|
||||
STD_TORCH_CHECK(output_n % 16 == 0,
|
||||
"The output tensor width must be a multiple of 16.");
|
||||
STD_TORCH_CHECK(
|
||||
input.scalar_type() == torch::headeronly::ScalarType::Half ||
|
||||
input.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||
@@ -197,7 +202,7 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
|
||||
input.get_device_index());
|
||||
auto stream = get_current_cuda_stream(input.get_device_index());
|
||||
|
||||
int sf_n_unpadded = int(n / CVT_FP4_SF_VEC_SIZE);
|
||||
int output_sf_n_unpadded = int(output_n / CVT_FP4_SF_VEC_SIZE);
|
||||
|
||||
// Grid, Block size. Each thread converts 8 values.
|
||||
dim3 block(std::min(int(n / ELTS_PER_THREAD), 512));
|
||||
@@ -205,7 +210,7 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
|
||||
vllm_runtime_blocks_per_sm(static_cast<int>(block.x));
|
||||
|
||||
if (is_sf_swizzled_layout) {
|
||||
int sf_n_int = int(vllm::round_up(sf_n_unpadded, 4) / 4);
|
||||
int sf_n_int = int(vllm::round_up(output_sf_n_unpadded, 4) / 4);
|
||||
int32_t num_padded_cols =
|
||||
sf_n_int * 4 * CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD;
|
||||
|
||||
@@ -220,12 +225,12 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
|
||||
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
|
||||
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
|
||||
vllm::cvt_fp16_to_fp4<cuda_type, false><<<grid, block, 0, stream>>>(
|
||||
m, n, num_padded_cols, input_ptr, input_sf_ptr,
|
||||
m, n, output_n, num_padded_cols, input_ptr, input_sf_ptr,
|
||||
reinterpret_cast<uint32_t*>(output_ptr),
|
||||
reinterpret_cast<uint32_t*>(sf_out));
|
||||
});
|
||||
} else {
|
||||
int num_packed_cols = n / CVT_FP4_ELTS_PER_THREAD;
|
||||
int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD;
|
||||
int grid_y = vllm::div_round_up(num_packed_cols, static_cast<int>(block.x));
|
||||
int grid_x = std::min(
|
||||
m, std::max(1, (multiProcessorCount * numBlocksPerSM) / grid_y));
|
||||
@@ -237,7 +242,8 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
|
||||
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
|
||||
vllm::cvt_fp16_to_fp4_sf_major<cuda_type, false>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
m, n, sf_n_unpadded, num_packed_cols, input_ptr, input_sf_ptr,
|
||||
m, n, output_n, output_sf_n_unpadded, num_packed_cols,
|
||||
input_ptr, input_sf_ptr,
|
||||
reinterpret_cast<uint32_t*>(output_ptr),
|
||||
reinterpret_cast<uint32_t*>(sf_out));
|
||||
});
|
||||
|
||||
@@ -34,6 +34,7 @@ PAD_SHAPES = [
|
||||
(64, 7152),
|
||||
(32, 14336),
|
||||
]
|
||||
PADDED_OUTPUT_SHAPES = [(128, 48), (128, 80), (150, 48), (150, 80), (64, 7152)]
|
||||
SEEDS = [42]
|
||||
CUDA_DEVICES = ["cuda:0"]
|
||||
|
||||
@@ -130,6 +131,10 @@ def recover_swizzled_scales(scale, m, n):
|
||||
return result[:m, :scale_n]
|
||||
|
||||
|
||||
def round_up(x: int, y: int) -> int:
|
||||
return (x + y - 1) // y * y
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("shape", SHAPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@@ -205,6 +210,60 @@ def test_python_util_matches_cpp_allocation(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", PADDED_OUTPUT_SHAPES)
|
||||
@pytest.mark.parametrize("is_sf_swizzled_layout", [True, False])
|
||||
@torch.inference_mode()
|
||||
def test_quantize_to_fp4_with_padded_output(
|
||||
shape: tuple[int, int],
|
||||
is_sf_swizzled_layout: bool,
|
||||
) -> None:
|
||||
from vllm._custom_ops import create_fp4_output_tensors
|
||||
|
||||
dtype = torch.float16
|
||||
set_random_seed(42)
|
||||
torch.set_default_device("cuda:0")
|
||||
|
||||
m, n = shape
|
||||
padded_n = round_up(n, 32)
|
||||
assert padded_n > n
|
||||
|
||||
x = torch.randn((m, n), dtype=dtype)
|
||||
tensor_amax = torch.abs(x).max().to(torch.float32)
|
||||
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
|
||||
out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
|
||||
|
||||
out, out_scale = ops.scaled_fp4_quant(
|
||||
x,
|
||||
global_scale,
|
||||
is_sf_swizzled_layout=is_sf_swizzled_layout,
|
||||
padded_n=padded_n,
|
||||
)
|
||||
py_out, py_scale = create_fp4_output_tensors(
|
||||
m,
|
||||
n,
|
||||
torch.device("cuda:0"),
|
||||
is_sf_swizzled_layout,
|
||||
padded_n=padded_n,
|
||||
)
|
||||
|
||||
assert out.shape == (m, padded_n // 2)
|
||||
assert out.shape == py_out.shape
|
||||
assert out_scale.shape == py_scale.view(torch.float8_e4m3fn).shape
|
||||
|
||||
out_ans = cast_from_fp4(out[:, : n // 2], m, n)
|
||||
torch.testing.assert_close(out_ans, out_ref)
|
||||
assert torch.count_nonzero(out[:, n // 2 :]) == 0
|
||||
|
||||
if is_sf_swizzled_layout:
|
||||
scale_ans = recover_swizzled_scales(out_scale, m, padded_n)
|
||||
torch.testing.assert_close(scale_ans[:, : n // BLOCK_SIZE], scale_ref)
|
||||
assert torch.count_nonzero(scale_ans[:, n // BLOCK_SIZE :]) == 0
|
||||
else:
|
||||
scale_ans = out_scale.to(torch.float32)
|
||||
torch.testing.assert_close(scale_ans[:, : n // BLOCK_SIZE], scale_ref)
|
||||
assert torch.count_nonzero(scale_ans[:, n // BLOCK_SIZE :]) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pad_shape", PAD_SHAPES)
|
||||
@torch.inference_mode()
|
||||
def test_quantize_to_fp4_padded(pad_shape: tuple[int, int]) -> None:
|
||||
|
||||
+24
-5
@@ -69,15 +69,20 @@ def create_fp4_output_tensors(
|
||||
n: int,
|
||||
device: torch.device,
|
||||
is_sf_swizzled_layout: bool,
|
||||
padded_n: int | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Allocate both output tensors for scaled_fp4_quant:
|
||||
(quantized_output, output_scale).
|
||||
|
||||
Must match the C++ scaled_fp4_quant_func allocation exactly.
|
||||
Must match the C++ scaled_fp4_quant_func allocation exactly when
|
||||
``padded_n`` is ``None``. When ``padded_n`` is provided, allocate a larger
|
||||
packed-FP4 output/scale buffer so the quantization kernel can write
|
||||
CUTLASS-compatible K padding directly
|
||||
"""
|
||||
output = torch.empty((m, n // 2), device=device, dtype=torch.uint8)
|
||||
output_scale = create_fp4_scale_tensor(m, n, device, is_sf_swizzled_layout)
|
||||
physical_n = padded_n if padded_n is not None else n
|
||||
output = torch.empty((m, physical_n // 2), device=device, dtype=torch.uint8)
|
||||
output_scale = create_fp4_scale_tensor(m, physical_n, device, is_sf_swizzled_layout)
|
||||
return output, output_scale
|
||||
|
||||
|
||||
@@ -1597,6 +1602,7 @@ def scaled_fp4_quant(
|
||||
input_global_scale: torch.Tensor,
|
||||
is_sf_swizzled_layout: bool = True,
|
||||
backend: str = "none",
|
||||
padded_n: int | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Quantize input tensor to FP4 and return quantized tensor and scale.
|
||||
@@ -1611,6 +1617,8 @@ def scaled_fp4_quant(
|
||||
input: The input tensor to be quantized to FP4
|
||||
input_global_scale: A scalar scaling factor for the entire tensor.
|
||||
use_8x4_sf_layout: Whether to use the 8x4 or 128x4 layout for the scaling
|
||||
padded_n: Optional padded K dimension. When provided, the quantized
|
||||
output and scale tensors are allocated for ``padded_n``
|
||||
|
||||
Returns:
|
||||
tuple[torch.Tensor, torch.Tensor]: The output tensor in FP4 but every
|
||||
@@ -1628,9 +1636,16 @@ def scaled_fp4_quant(
|
||||
assert input.dtype in (torch.float16, torch.bfloat16), (
|
||||
f"input.dtype needs to be fp16 or bf16 but got {input.dtype}."
|
||||
)
|
||||
if padded_n is not None:
|
||||
assert padded_n >= n, f"padded_n must be >= n, got padded_n={padded_n}, n={n}."
|
||||
assert padded_n % block_size == 0, (
|
||||
f"padded_n has to be a multiple of {block_size}, but got {padded_n}."
|
||||
)
|
||||
|
||||
use_8x4_sf_layout = True if "trtllm" in backend and m <= 32 else False # noqa: SIM210
|
||||
|
||||
if use_8x4_sf_layout and padded_n is not None and padded_n != n:
|
||||
# TODO: support this case
|
||||
raise ValueError("padded_n is not supported with TRTLLM 8x4 scale layout.")
|
||||
if use_8x4_sf_layout:
|
||||
output, output_scale = flashinfer_quant_nvfp4_8x4_sf_layout(
|
||||
input, input_global_scale
|
||||
@@ -1638,7 +1653,11 @@ def scaled_fp4_quant(
|
||||
else:
|
||||
# Pre-allocate and call .out variant (same behavior as old in-place API)
|
||||
output, output_scale = create_fp4_output_tensors(
|
||||
m, n, input.device, is_sf_swizzled_layout
|
||||
m,
|
||||
n,
|
||||
input.device,
|
||||
is_sf_swizzled_layout,
|
||||
padded_n=padded_n,
|
||||
)
|
||||
torch.ops._C.scaled_fp4_quant.out(
|
||||
input,
|
||||
|
||||
@@ -9,7 +9,6 @@ from vllm._custom_ops import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
cutlass_fp4_supported,
|
||||
pad_nvfp4_activation_for_cutlass,
|
||||
pad_nvfp4_weight_for_cutlass,
|
||||
slice_nvfp4_output,
|
||||
swizzle_blockscale,
|
||||
@@ -52,16 +51,14 @@ class CutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
output_size = layer.output_size_per_partition
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
|
||||
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
layer.input_global_scale_inv,
|
||||
is_sf_swizzled_layout=True,
|
||||
backend="cutlass",
|
||||
)
|
||||
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(
|
||||
x_fp4, getattr(layer, "weights_padding_cols", 0)
|
||||
padded_n=x.shape[-1] + weights_padding_bytes * 2,
|
||||
)
|
||||
|
||||
out = cutlass_scaled_fp4_mm(
|
||||
|
||||
@@ -5,7 +5,6 @@ import torch
|
||||
|
||||
from vllm._custom_ops import scaled_fp4_quant
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
pad_nvfp4_activation_for_cutlass,
|
||||
pad_nvfp4_weight_for_cutlass,
|
||||
slice_nvfp4_output,
|
||||
swizzle_blockscale,
|
||||
@@ -58,16 +57,14 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
output_size = layer.output_size_per_partition
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
|
||||
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
layer.input_global_scale_inv,
|
||||
is_sf_swizzled_layout=True,
|
||||
backend="flashinfer-cutlass",
|
||||
)
|
||||
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(
|
||||
x_fp4, getattr(layer, "weights_padding_cols", 0)
|
||||
padded_n=x.shape[-1] + weights_padding_bytes * 2,
|
||||
)
|
||||
|
||||
out = flashinfer_scaled_fp4_mm(
|
||||
@@ -189,16 +186,14 @@ class FlashInferCudnnNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
output_size = layer.output_size_per_partition
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
|
||||
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
layer.input_global_scale_inv,
|
||||
is_sf_swizzled_layout=True,
|
||||
backend="flashinfer-cudnn",
|
||||
)
|
||||
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(
|
||||
x_fp4, getattr(layer, "weights_padding_cols", 0)
|
||||
padded_n=x.shape[-1] + weights_padding_bytes * 2,
|
||||
)
|
||||
|
||||
out = flashinfer_scaled_fp4_mm(
|
||||
|
||||
Reference in New Issue
Block a user