[Perf] per_token_group_quant_8bit: multiply by reciprocal instead of dividing per element (#61343)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Thien Tran
2026-10-13 21:37:10 +08:00
committed by GitHub
parent 31e90ac370
commit 0fae39a014
2 changed files with 15 additions and 9 deletions
@@ -516,6 +516,11 @@ __device__ void norm_and_quant(
q8x4_t<scalar_out_t> out;
// Group quant multiplies by the reciprocal of the scale (one div per
// group), matching per_token_group_quant_8bit. Per-token fp8 must keep
// exact division for FBGemm parity, so it is excluded here.
constexpr bool mul_by_recip = (group_size > 0) || is_scale_inverted;
float scale_val;
if constexpr (group_size > 0) {
@@ -528,14 +533,13 @@ __device__ void norm_and_quant(
} else {
scale_idx = blockIdx.x * num_groups + i * VEC_SIZE / group_size;
}
scale_val =
is_scale_inverted ? 1.0f / scale[scale_idx] : scale[scale_idx];
scale_val = 1.0f / scale[scale_idx];
} else {
scale_val = *scale;
}
#pragma unroll
for (int j = 0; j < VEC_SIZE; ++j) {
out.val[j] = ScaledQuant<scalar_out_t, is_scale_inverted>::quant_fn(
out.val[j] = ScaledQuant<scalar_out_t, mul_by_recip>::quant_fn(
static_cast<scalar_t>(x.val[j] * rms) * w.val[j], scale_val);
}
vec_output[i] = out;