[Perf][Kernel] Initialize NVFP4 padding in quant kernel (#53568)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
cdefd9d499
commit
514c7314a0
@@ -36,7 +36,7 @@
|
||||
namespace vllm {
|
||||
|
||||
// Use UE4M3 by default.
|
||||
template <class Type, bool UE8M0_SF = false>
|
||||
template <class Type, bool UE8M0_SF = false, bool PADDED_OUTPUT = false>
|
||||
__global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
cvt_fp16_to_fp4(int32_t numRows, int32_t numCols, int32_t outputCols,
|
||||
int32_t num_padded_cols, Type const* __restrict__ in,
|
||||
@@ -55,7 +55,7 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
#endif
|
||||
|
||||
// Precompute SF layout parameter (constant for entire kernel).
|
||||
int32_t const numKTiles = (outputCols + 63) / 64;
|
||||
int32_t const numKTiles = ((PADDED_OUTPUT ? outputCols : numCols) + 63) / 64;
|
||||
|
||||
int sf_m = round_up<int>(numRows, 128);
|
||||
int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x;
|
||||
@@ -74,16 +74,15 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
int64_t inOffset = rowIdx * (numCols / CVT_FP4_ELTS_PER_THREAD) + colIdx;
|
||||
|
||||
// If we are outside valid columns, feed zeros
|
||||
bool valid_input = (rowIdx < numRows) && (elem_idx < numCols);
|
||||
bool valid_output = (rowIdx < numRows) && (elem_idx < outputCols);
|
||||
bool valid = (rowIdx < numRows) && (elem_idx < numCols);
|
||||
if constexpr (CVT_FP4_PACK16) {
|
||||
ld256_cg_or_zero(reinterpret_cast<u32x8_t&>(in_vec),
|
||||
&reinterpret_cast<const uint32_t*>(in)[inOffset * 8],
|
||||
valid_input);
|
||||
valid);
|
||||
} else {
|
||||
ld128_cg_or_zero(reinterpret_cast<uint4&>(in_vec),
|
||||
&reinterpret_cast<const uint32_t*>(in)[inOffset * 4],
|
||||
valid_input);
|
||||
valid);
|
||||
}
|
||||
|
||||
auto sf_out =
|
||||
@@ -91,20 +90,42 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
CVT_FP4_NUM_THREADS_PER_SF>(
|
||||
rowIdx, colIdx, numKTiles, SFout);
|
||||
|
||||
auto out_val =
|
||||
cvt_warp_fp16_to_fp4<Type, CVT_FP4_NUM_THREADS_PER_SF, UE8M0_SF>(
|
||||
in_vec, global_scale, sf_out);
|
||||
fp4_packed_t out_val{};
|
||||
if (valid) {
|
||||
out_val =
|
||||
cvt_warp_fp16_to_fp4<Type, CVT_FP4_NUM_THREADS_PER_SF, UE8M0_SF>(
|
||||
in_vec, global_scale, sf_out);
|
||||
} else if (sf_out != nullptr) {
|
||||
// The swizzled scale tensor is allocated with torch.empty. Explicitly
|
||||
// initialize padded rows and columns in this kernel so downstream
|
||||
// NVFP4 GEMMs never observe stale scale factors.
|
||||
*sf_out = 0;
|
||||
}
|
||||
|
||||
if (valid_output) {
|
||||
if constexpr (PADDED_OUTPUT) {
|
||||
bool valid_output = (rowIdx < numRows) && (elem_idx < outputCols);
|
||||
if (valid_output) {
|
||||
if constexpr (CVT_FP4_PACK16) {
|
||||
int64_t outOffset = rowIdx * (outputCols / 8) + colIdx * 2;
|
||||
uint64_t packed64 =
|
||||
(uint64_t(out_val.hi) << 32) | uint64_t(out_val.lo);
|
||||
reinterpret_cast<uint64_t*>(out)[outOffset >> 1] = packed64;
|
||||
} else {
|
||||
int64_t outOffset =
|
||||
rowIdx * (outputCols / CVT_FP4_ELTS_PER_THREAD) + colIdx;
|
||||
reinterpret_cast<fp4_packed_t*>(out)[outOffset] = out_val;
|
||||
}
|
||||
}
|
||||
} else if (valid) {
|
||||
// #32520 fast path: input and output widths are identical, so avoid
|
||||
// the padded-output predicate and use the original address arithmetic.
|
||||
if constexpr (CVT_FP4_PACK16) {
|
||||
int64_t outOffset = rowIdx * (outputCols / 8) + colIdx * 2;
|
||||
int64_t outOffset = rowIdx * (numCols / 8) + colIdx * 2;
|
||||
uint64_t packed64 =
|
||||
(uint64_t(out_val.hi) << 32) | uint64_t(out_val.lo);
|
||||
reinterpret_cast<uint64_t*>(out)[outOffset >> 1] = packed64;
|
||||
} else {
|
||||
int64_t outOffset =
|
||||
rowIdx * (outputCols / CVT_FP4_ELTS_PER_THREAD) + colIdx;
|
||||
out[outOffset] = out_val;
|
||||
reinterpret_cast<fp4_packed_t*>(out)[inOffset] = out_val;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -244,11 +265,19 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = 1;
|
||||
config.numAttrs = 1;
|
||||
config.attrs = attrs;
|
||||
cudaLaunchKernelEx(&config, vllm::cvt_fp16_to_fp4<cuda_type, false>,
|
||||
m, n, output_n, num_padded_cols, input_ptr,
|
||||
input_sf_ptr,
|
||||
reinterpret_cast<uint32_t*>(output_ptr),
|
||||
reinterpret_cast<uint32_t*>(sf_out));
|
||||
if (output_n == n) {
|
||||
cudaLaunchKernelEx(
|
||||
&config, vllm::cvt_fp16_to_fp4<cuda_type, false, false>, m, n,
|
||||
output_n, num_padded_cols, input_ptr, input_sf_ptr,
|
||||
reinterpret_cast<uint32_t*>(output_ptr),
|
||||
reinterpret_cast<uint32_t*>(sf_out));
|
||||
} else {
|
||||
cudaLaunchKernelEx(
|
||||
&config, vllm::cvt_fp16_to_fp4<cuda_type, false, true>, m, n,
|
||||
output_n, num_padded_cols, input_ptr, input_sf_ptr,
|
||||
reinterpret_cast<uint32_t*>(output_ptr),
|
||||
reinterpret_cast<uint32_t*>(sf_out));
|
||||
}
|
||||
});
|
||||
} else {
|
||||
int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD;
|
||||
|
||||
@@ -119,7 +119,7 @@ def ref_nvfp4_quant(x, global_scale):
|
||||
return cast_to_fp4(clipped_x), scale.squeeze(-1)
|
||||
|
||||
|
||||
def recover_swizzled_scales(scale, m, n):
|
||||
def recover_swizzled_scales(scale, m, n, include_padding=False):
|
||||
round_up = lambda x, y: (x + y - 1) // y * y
|
||||
rounded_m = round_up(m, 128)
|
||||
scale_n = n // BLOCK_SIZE
|
||||
@@ -128,6 +128,8 @@ def recover_swizzled_scales(scale, m, n):
|
||||
tmp = torch.reshape(scale, (1, rounded_m // 128, rounded_n // 4, 32, 4, 4))
|
||||
tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5))
|
||||
result = torch.reshape(tmp, (rounded_m, rounded_n)).to(torch.float32)
|
||||
if include_padding:
|
||||
return result
|
||||
return result[:m, :scale_n]
|
||||
|
||||
|
||||
@@ -264,6 +266,43 @@ def test_quantize_to_fp4_with_padded_output(
|
||||
assert torch.count_nonzero(scale_ans[:, n // BLOCK_SIZE :]) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("shape", "padded_n"),
|
||||
[((90, 80), None), ((90, 48), 64)],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_swizzled_scale_padding_initialized_in_kernel(
|
||||
shape: tuple[int, int], padded_n: int | None
|
||||
) -> None:
|
||||
"""The out variant must overwrite dirty scale padding without a memset."""
|
||||
torch.set_default_device("cuda:0")
|
||||
m, n = shape
|
||||
physical_n = padded_n if padded_n is not None else n
|
||||
rounded_m = round_up(m, 128)
|
||||
rounded_scale_n = round_up(physical_n // BLOCK_SIZE, 4)
|
||||
|
||||
x = torch.randn((m, n), dtype=torch.float16)
|
||||
global_scale = torch.tensor(1.0, dtype=torch.float32)
|
||||
out = torch.full((m, physical_n // 2), 0xFF, dtype=torch.uint8)
|
||||
out_scale = torch.full(
|
||||
(rounded_m, rounded_scale_n // 4), 0x7F7F7F7F, dtype=torch.int32
|
||||
)
|
||||
|
||||
torch.ops._C.scaled_fp4_quant.out(
|
||||
x,
|
||||
global_scale,
|
||||
True,
|
||||
output=out,
|
||||
output_scale=out_scale,
|
||||
)
|
||||
|
||||
full_scale = recover_swizzled_scales(
|
||||
out_scale.view(torch.float8_e4m3fn), m, physical_n, include_padding=True
|
||||
)
|
||||
assert torch.count_nonzero(full_scale[m:, :]) == 0
|
||||
assert torch.count_nonzero(full_scale[:m, n // BLOCK_SIZE :]) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pad_shape", PAD_SHAPES)
|
||||
@torch.inference_mode()
|
||||
def test_quantize_to_fp4_padded(pad_shape: tuple[int, int]) -> None:
|
||||
|
||||
+3
-7
@@ -57,13 +57,9 @@ def create_fp4_scale_tensor(
|
||||
rounded_m = round_up(m, 128)
|
||||
scale_n = n // block_size
|
||||
rounded_n = round_up(scale_n, 4)
|
||||
# Must be zero-initialized: the swizzled scale buffer is padded to
|
||||
# (round_up(m, 128), round_up(scale_n, 4) // 4) but the NVFP4 quant
|
||||
# kernel does not write every padded element that the downstream
|
||||
# NVFP4 GEMM reads. torch.empty leaves those padded scale factors
|
||||
# uninitialized, which corrupts dequantization and causes a severe
|
||||
# Blackwell NVFP4 decode throughput/output-length regression.
|
||||
return torch.zeros(
|
||||
# The NVFP4 quant kernel explicitly zeroes every padded scale entry,
|
||||
# so no separate zero-initialization kernel is required here.
|
||||
return torch.empty(
|
||||
(rounded_m, rounded_n // 4), device=device, dtype=torch.int32
|
||||
)
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user