diff --git a/csrc/libtorch_stable/quantization/fused_kernels/layernorm_utils.cuh b/csrc/libtorch_stable/quantization/fused_kernels/layernorm_utils.cuh index 0139cfbc95..9a3d221562 100644 --- a/csrc/libtorch_stable/quantization/fused_kernels/layernorm_utils.cuh +++ b/csrc/libtorch_stable/quantization/fused_kernels/layernorm_utils.cuh @@ -47,27 +47,13 @@ __device__ void compute_rms(float* rms, scalar_t const* __restrict__ input, *rms = s_rms; } -__device__ float warpReduceMaxSpecialized(volatile float* val, int64_t tid, - int64_t thread_in_warp, - int64_t reduced_elems) { +__device__ float warpReduceMax(float val) { static_assert(WARP_SIZE == 32 || WARP_SIZE == 64); - if constexpr (WARP_SIZE == 64) { - if (thread_in_warp + 64 < reduced_elems) - val[tid] = fmaxf(val[tid], val[tid + 64]); +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { + val = fmaxf(val, VLLM_SHFL_DOWN_SYNC(val, offset)); } - if (thread_in_warp + 32 < reduced_elems) - val[tid] = fmaxf(val[tid], val[tid + 32]); - if (thread_in_warp + 16 < reduced_elems) - val[tid] = fmaxf(val[tid], val[tid + 16]); - if (thread_in_warp + 8 < reduced_elems) - val[tid] = fmaxf(val[tid], val[tid + 8]); - if (thread_in_warp + 4 < reduced_elems) - val[tid] = fmaxf(val[tid], val[tid + 4]); - if (thread_in_warp + 2 < reduced_elems) - val[tid] = fmaxf(val[tid], val[tid + 2]); - if (thread_in_warp + 1 < reduced_elems) - val[tid] = fmaxf(val[tid], val[tid + 1]); - return val[tid]; + return val; } template (hidden_size)); - for (auto j = start; j + warp_size < warp_end; j += warp_size) { - s_max_vals[start] = - fmaxf(s_max_vals[start], s_max_vals[j + warp_size]); + int64_t const warp_start = group_id * threads_per_group; + float warp_max = 0.0f; + for (auto j = thread_in_warp; j < threads_per_group; j += warp_size) { + warp_max = fmaxf(warp_max, s_max_vals[warp_start + j]); + } + warp_max = warpReduceMax(warp_max); + if (thread_in_warp == 0) { + s_max_vals[warp_start] = warp_max; } - warpReduceMaxSpecialized(s_max_vals, start, thread_in_warp, - min(warp_end - warp_start, warp_size)); } } __syncthreads(); @@ -375,16 +360,15 @@ __device__ void compute_dynamic_per_token_scales( for (auto i = 0; i < groups_per_warp; ++i) { int64_t const group_id = i * num_warps + warp_id; if (group_id < num_groups) { - int64_t warp_start = group_id * threads_per_group; - int64_t const start = warp_start + thread_in_warp; - int64_t const warp_end = min(warp_start + threads_per_group, - static_cast(hidden_size)); - for (auto j = start; j + warp_size < warp_end; j += warp_size) { - s_max_vals[start] = - fmaxf(s_max_vals[start], s_max_vals[j + warp_size]); + int64_t const warp_start = group_id * threads_per_group; + float warp_max = 0.0f; + for (auto j = thread_in_warp; j < threads_per_group; j += warp_size) { + warp_max = fmaxf(warp_max, s_max_vals[warp_start + j]); + } + warp_max = warpReduceMax(warp_max); + if (thread_in_warp == 0) { + s_max_vals[warp_start] = warp_max; } - warpReduceMaxSpecialized(s_max_vals, start, thread_in_warp, - min(warp_end - warp_start, warp_size)); } } __syncthreads(); diff --git a/tests/kernels/core/test_fused_quant_layernorm.py b/tests/kernels/core/test_fused_quant_layernorm.py index 06df13d48c..201830fe2f 100644 --- a/tests/kernels/core/test_fused_quant_layernorm.py +++ b/tests/kernels/core/test_fused_quant_layernorm.py @@ -330,6 +330,12 @@ def test_rms_norm( and dtype == torch.bfloat16 and quant_dtype == current_platform.fp8_dtype() ) + allow_cuda_fp8_rounding_outliers = ( + current_platform.is_cuda() + and group_size is None + and dtype == torch.bfloat16 + and quant_dtype == current_platform.fp8_dtype() + ) def scales_close(rtol: float, atol: float) -> bool: if torch.allclose(ref_scales, ops_scales, rtol=rtol, atol=atol): @@ -358,8 +364,15 @@ def test_rms_norm( # Valid gfx950 reduction trees can straddle an E4M3 boundary. ok = fp8_allclose(ops_out, ref_out, rtol=0.125, atol=2e-3) ok = ok and int(fp8_ulp_distance(ops_out, ref_out).max()) <= 1 + elif allow_cuda_fp8_rounding_outliers: + # A valid BF16 reduction can cross an E4M3 boundary for + # isolated values. + ulp = fp8_ulp_distance(ref_out, ops_out) + max_outliers = ulp.numel() // 100_000 + 8 + ok = int(ulp.max()) <= 1 + ok = ok and int((ulp > 0).sum().item()) <= max_outliers else: - # CUDA (& non-bf16): compare dequantized values with relaxed tolerance. + # Compare dequantized values with relaxed tolerance. if group_size is None: a_deq = a * ref_scales.view(-1, 1) b_deq = b * ops_scales.view(-1, 1)