[Bugfix][Quantization] Clamp FP8 scales before UE8M0 rounding (#58032)

Signed-off-by: Ashish Rao <asrao@nvidia.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
This commit is contained in:
Ashish Rao
2026-10-14 13:56:42 +00:00
committed by GitHub
co-authored by Vadim Gimpelson
parent 7115b70c01
commit b4b7eca0ef
2 changed files with 29 additions and 5 deletions
@@ -244,10 +244,12 @@ class QuantFP8(CustomOp):
x_grouped = x.view(-1, num_groups, self.group_size)
absmax = x_grouped.abs().max(dim=-1, keepdim=True)[0].float()
scales_raw = absmax / _FP8_MAX
if self.use_ue8m0:
scales_raw = torch.exp2(torch.ceil(torch.log2(scales_raw)))
scales = (scales_raw).clamp(min=_FP8_MIN_SCALING_FACTOR)
scales_raw = (absmax / _FP8_MAX).clamp(min=_FP8_MIN_SCALING_FACTOR)
scales = (
torch.exp2(torch.ceil(torch.log2(scales_raw)))
if self.use_ue8m0
else scales_raw
)
x_scaled = x_grouped / scales
x_quant = x_scaled.clamp(_FP8_MIN, _FP8_MAX).to(_FP8_DTYPE)