Add NVFP4 KV 4-over-6 scale search (#45187)

Signed-off-by: weimingc <17592131+meenchen@users.noreply.github.com>
This commit is contained in:
Wei-Ming Chen
2026-08-07 13:22:27 -07:00
committed by GitHub
parent 70456e5e6f
commit 1c94e8dc7d
11 changed files with 284 additions and 42 deletions
+4 -3
View File
@@ -770,16 +770,17 @@ void reshape_and_cache_flash(
key.get_device_index());
const cudaStream_t stream = get_current_cuda_stream();
if (kv_cache_dtype == "nvfp4") {
if (kv_cache_dtype == "nvfp4" || kv_cache_dtype == "nvfp4_4over6") {
#if defined(ENABLE_NVFP4_SM100) || defined(ENABLE_NVFP4_SM120)
// NVFP4 dispatch is compiled separately for SM100+.
extern void reshape_and_cache_nvfp4_dispatch(
torch::stable::Tensor & key, torch::stable::Tensor & value,
torch::stable::Tensor & key_cache, torch::stable::Tensor & value_cache,
torch::stable::Tensor & slot_mapping, torch::stable::Tensor & k_scale,
torch::stable::Tensor & v_scale);
torch::stable::Tensor & v_scale, const std::string& kv_cache_dtype);
reshape_and_cache_nvfp4_dispatch(key, value, key_cache, value_cache,
slot_mapping, k_scale, v_scale);
slot_mapping, k_scale, v_scale,
kv_cache_dtype);
return;
#else
STD_TORCH_CHECK(
+167 -21
View File
@@ -20,8 +20,17 @@
#include "libtorch_stable/dispatch_utils.h"
#include "libtorch_stable/torch_utils.h"
#include <cmath>
#include <string>
namespace vllm {
// The cache layout is identical; this only selects the store-time scale search.
enum class NVFP4KVScaleSearch {
DEFAULT,
FOUR_OVER_SIX,
};
// Compute swizzled scale offset for SM100 trtllm-gen MHA kernel.
// The swizzle pattern for HND layout is:
// [T//4, 4, 4, S//4] → permute(0, 2, 3, 1) → reshape to [T, S]
@@ -38,6 +47,118 @@ __device__ __forceinline__ int swizzle_scale_offset(int t, int s,
return swizzled_t * scale_dim + swizzled_s;
}
__device__ __forceinline__ float round_to_nearest_e2m1(float x) {
const float ax = fabsf(x);
float q;
// Match cvt.rn.satfinite.e2m1x2.f32, including ties-to-even boundaries.
if (ax <= 0.25f) {
q = 0.0f;
} else if (ax < 0.75f) {
q = 0.5f;
} else if (ax <= 1.25f) {
q = 1.0f;
} else if (ax < 1.75f) {
q = 1.5f;
} else if (ax <= 2.5f) {
q = 2.0f;
} else if (ax < 3.5f) {
q = 3.0f;
} else if (ax <= 5.0f) {
q = 4.0f;
} else {
q = 6.0f;
}
return copysignf(q, x);
}
__device__ __forceinline__ void nvfp4_candidate_scale(float SFScaleVal,
float vecMax,
float denominator,
uint8_t* fp8_sf,
float* outputScale) {
float SFValue =
SFScaleVal * (vecMax * reciprocal_approximate_ftz(denominator));
__nv_fp8_e4m3 tmp = __nv_fp8_e4m3(SFValue);
reinterpret_cast<__nv_fp8_e4m3&>(*fp8_sf) = tmp;
SFValue = float(tmp);
*outputScale = SFValue != 0.0f
? reciprocal_approximate_ftz(
SFValue * reciprocal_approximate_ftz(SFScaleVal))
: 0.0f;
}
template <class Type, int CVT_FP4_NUM_THREADS_PER_SF>
__device__ __forceinline__ float nvfp4_reconstruction_error(
PackedVec<Type, CVT_FP4_PACK16>& vec, float outputScale) {
float error = 0.0f;
const float dequantScale =
outputScale != 0.0f ? reciprocal_approximate_ftz(outputScale) : 0.0f;
#pragma unroll
for (int i = 0; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) {
float2 vals = cast_to_float2(vec.elts[i]);
if (outputScale == 0.0f) {
error += vals.x * vals.x + vals.y * vals.y;
} else {
float qx = round_to_nearest_e2m1(vals.x * outputScale);
float qy = round_to_nearest_e2m1(vals.y * outputScale);
float dx = qx * dequantScale - vals.x;
float dy = qy * dequantScale - vals.y;
error += dx * dx + dy * dy;
}
}
if constexpr (CVT_FP4_NUM_THREADS_PER_SF == 2) {
error += __shfl_xor_sync(0xffffffffu, error, 1);
}
return error;
}
template <class Type, int CVT_FP4_NUM_THREADS_PER_SF>
__device__ __forceinline__ fp4_packed_t cvt_warp_fp16_to_fp4_4over6(
PackedVec<Type, CVT_FP4_PACK16>& vec, float SFScaleVal, uint8_t* SFout) {
auto localMax = __habs2(vec.elts[0]);
#pragma unroll
for (int i = 1; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) {
localMax = __hmax2(localMax, __habs2(vec.elts[i]));
}
if constexpr (CVT_FP4_NUM_THREADS_PER_SF == 2) {
localMax = __hmax2(__shfl_xor_sync(0xffffffffu, localMax, 1), localMax);
}
const float vecMax = float(__hmax(localMax.x, localMax.y));
uint8_t sf6;
uint8_t sf4;
float outputScale6;
float outputScale4;
nvfp4_candidate_scale(SFScaleVal, vecMax, 6.0f, &sf6, &outputScale6);
nvfp4_candidate_scale(SFScaleVal, vecMax, 4.0f, &sf4, &outputScale4);
const float err6 =
nvfp4_reconstruction_error<Type, CVT_FP4_NUM_THREADS_PER_SF>(
vec, outputScale6);
const float err4 =
nvfp4_reconstruction_error<Type, CVT_FP4_NUM_THREADS_PER_SF>(
vec, outputScale4);
const bool use4 = err4 < err6;
const float outputScale = use4 ? outputScale4 : outputScale6;
if (SFout) *SFout = use4 ? sf4 : sf6;
float2 fp2Vals[CVT_FP4_ELTS_PER_THREAD / 2];
#pragma unroll
for (int i = 0; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) {
fp2Vals[i] = cast_to_float2(vec.elts[i]);
fp2Vals[i].x *= outputScale;
fp2Vals[i].y *= outputScale;
}
return pack_fp4(fp2Vals);
}
// Kernel: quantize bf16 key/value to NVFP4 and store in paged KV cache.
//
// Takes separate data and scale cache pointers for K and V.
@@ -45,7 +166,7 @@ __device__ __forceinline__ int swizzle_scale_offset(int t, int s,
//
// Threading: one CUDA block per token, threads process heads and
// groups of 16 elements within each head.
template <typename scalar_t>
template <typename scalar_t, NVFP4KVScaleSearch SCALE_SEARCH>
__global__ void reshape_and_cache_nvfp4_kernel(
const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
@@ -129,8 +250,14 @@ __global__ void reshape_and_cache_nvfp4_kernel(
uint8_t sf_val;
uint8_t* sf_out_ptr = (tg_lane == 0) ? &sf_val : nullptr;
fp4_packed_t packed = cvt_warp_fp16_to_fp4<CudaType, THREADS_PER_SF>(
in_vec, global_scale, sf_out_ptr);
fp4_packed_t packed;
if constexpr (SCALE_SEARCH == NVFP4KVScaleSearch::FOUR_OVER_SIX) {
packed = cvt_warp_fp16_to_fp4_4over6<CudaType, THREADS_PER_SF>(
in_vec, global_scale, sf_out_ptr);
} else {
packed = cvt_warp_fp16_to_fp4<CudaType, THREADS_PER_SF>(
in_vec, global_scale, sf_out_ptr);
}
// Write packed FP4 data to data cache
uint8_t* __restrict__ data_dst = data_block + head * data_head_stride +
@@ -181,13 +308,11 @@ __global__ void reshape_and_cache_nvfp4_kernel(
// Receives key_cache/value_cache as kv_cache[:, 0] and kv_cache[:, 1].
// Each KV side contains both data and scale:
// page = [K_data | K_scale | V_data | V_scale]
void reshape_and_cache_nvfp4_dispatch(torch::stable::Tensor& key,
torch::stable::Tensor& value,
torch::stable::Tensor& key_cache,
torch::stable::Tensor& value_cache,
torch::stable::Tensor& slot_mapping,
torch::stable::Tensor& k_scale,
torch::stable::Tensor& v_scale) {
void reshape_and_cache_nvfp4_dispatch(
torch::stable::Tensor& key, torch::stable::Tensor& value,
torch::stable::Tensor& key_cache, torch::stable::Tensor& value_cache,
torch::stable::Tensor& slot_mapping, torch::stable::Tensor& k_scale,
torch::stable::Tensor& v_scale, const std::string& kv_cache_dtype) {
int num_tokens = slot_mapping.size(0);
int num_heads = key.size(1);
int head_size = key.size(2);
@@ -262,16 +387,37 @@ void reshape_and_cache_nvfp4_dispatch(torch::stable::Tensor& key,
VLLM_STABLE_DISPATCH_HALF_TYPES(
key.scalar_type(), "reshape_and_cache_nvfp4", [&] {
vllm::reshape_and_cache_nvfp4_kernel<scalar_t>
<<<grid, block, 0, stream>>>(
key.const_data_ptr<scalar_t>(),
value.const_data_ptr<scalar_t>(),
key_cache.mutable_data_ptr<uint8_t>(),
value_cache.mutable_data_ptr<uint8_t>(), key_scale_ptr,
value_scale_ptr, slot_mapping.const_data_ptr<int64_t>(),
k_scale_ptr, v_scale_ptr, key.stride(0), value.stride(0),
num_heads, head_size, block_size, data_block_stride,
data_head_stride, data_block_offset_stride, scale_block_stride,
scale_head_stride, scale_block_offset_stride);
if (kv_cache_dtype == "nvfp4") {
vllm::reshape_and_cache_nvfp4_kernel<
scalar_t, vllm::NVFP4KVScaleSearch::DEFAULT>
<<<grid, block, 0, stream>>>(
key.const_data_ptr<scalar_t>(),
value.const_data_ptr<scalar_t>(),
key_cache.mutable_data_ptr<uint8_t>(),
value_cache.mutable_data_ptr<uint8_t>(), key_scale_ptr,
value_scale_ptr, slot_mapping.const_data_ptr<int64_t>(),
k_scale_ptr, v_scale_ptr, key.stride(0), value.stride(0),
num_heads, head_size, block_size, data_block_stride,
data_head_stride, data_block_offset_stride,
scale_block_stride, scale_head_stride,
scale_block_offset_stride);
} else if (kv_cache_dtype == "nvfp4_4over6") {
vllm::reshape_and_cache_nvfp4_kernel<
scalar_t, vllm::NVFP4KVScaleSearch::FOUR_OVER_SIX>
<<<grid, block, 0, stream>>>(
key.const_data_ptr<scalar_t>(),
value.const_data_ptr<scalar_t>(),
key_cache.mutable_data_ptr<uint8_t>(),
value_cache.mutable_data_ptr<uint8_t>(), key_scale_ptr,
value_scale_ptr, slot_mapping.const_data_ptr<int64_t>(),
k_scale_ptr, v_scale_ptr, key.stride(0), value.stride(0),
num_heads, head_size, block_size, data_block_stride,
data_head_stride, data_block_offset_stride,
scale_block_stride, scale_head_stride,
scale_block_offset_stride);
} else {
STD_TORCH_CHECK(false,
"Unsupported NVFP4 KV cache dtype: ", kv_cache_dtype);
}
});
}