[Perf] per_token_group_quant_8bit: multiply by reciprocal instead of dividing per element (#61343)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
@@ -516,6 +516,11 @@ __device__ void norm_and_quant(
|
||||
|
||||
q8x4_t<scalar_out_t> out;
|
||||
|
||||
// Group quant multiplies by the reciprocal of the scale (one div per
|
||||
// group), matching per_token_group_quant_8bit. Per-token fp8 must keep
|
||||
// exact division for FBGemm parity, so it is excluded here.
|
||||
constexpr bool mul_by_recip = (group_size > 0) || is_scale_inverted;
|
||||
|
||||
float scale_val;
|
||||
|
||||
if constexpr (group_size > 0) {
|
||||
@@ -528,14 +533,13 @@ __device__ void norm_and_quant(
|
||||
} else {
|
||||
scale_idx = blockIdx.x * num_groups + i * VEC_SIZE / group_size;
|
||||
}
|
||||
scale_val =
|
||||
is_scale_inverted ? 1.0f / scale[scale_idx] : scale[scale_idx];
|
||||
scale_val = 1.0f / scale[scale_idx];
|
||||
} else {
|
||||
scale_val = *scale;
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < VEC_SIZE; ++j) {
|
||||
out.val[j] = ScaledQuant<scalar_out_t, is_scale_inverted>::quant_fn(
|
||||
out.val[j] = ScaledQuant<scalar_out_t, mul_by_recip>::quant_fn(
|
||||
static_cast<scalar_t>(x.val[j] * rms) * w.val[j], scale_val);
|
||||
}
|
||||
vec_output[i] = out;
|
||||
|
||||
Reference in New Issue
Block a user