[Quantization] Preserve precision in online NVFP4 expert packing (#50029)

Signed-off-by: S1ro1 <matej.sirovatka@gmail.com>
This commit is contained in:
Matej Sirovatka
2026-08-06 02:23:54 -07:00
committed by GitHub
parent 276f0bb5c1
commit 9c22668436
2 changed files with 58 additions and 10 deletions
@@ -40,23 +40,34 @@ def _quantize_moe_weight_to_nvfp4(
global scale ``(E,)`` stored as ``amax / (fp4_max * fp8_max)``.
"""
assert weight.dim() == 3, f"expected 3D expert weights, got {weight.shape}"
num_experts, n, k = weight.shape
k = weight.shape[-1]
assert k % 16 == 0, f"last dim must be a multiple of 16, got {k}"
amax = weight.abs().amax(dim=(1, 2)).to(torch.float32).clamp_min(1e-8)
global_scale = (FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX) / amax
weight_scale_2 = (1.0 / global_scale).to(torch.float32)
# scaled_fp4_quant(w, g) == scaled_fp4_quant(w * g, 1), so fold each
# expert's scale in and quantize all experts in one call (fp32 to keep the
# large scale precise), rather than looping per expert.
scaled = (weight.float() * global_scale[:, None, None]).to(weight.dtype)
scaled = scaled.reshape(-1, k)
one = torch.ones((), device=weight.device, dtype=torch.float32)
qweight, block_scale = scaled_fp4_quant(scaled, one, is_sf_swizzled_layout=False)
# Keep the original BF16/FP16 values as the quantizer input. Folding each
# expert's FP32 global scale into the weight would add a BF16/FP16 rounding
# before the group-16 scale and E2M1 values are selected.
weight = weight.contiguous()
quantized_experts = [
scaled_fp4_quant(
expert_weight,
expert_scale,
is_sf_swizzled_layout=False,
)
for expert_weight, expert_scale in zip(
weight,
global_scale,
strict=True,
)
]
qweight = torch.stack([quantized for quantized, _ in quantized_experts])
block_scale = torch.stack([block_scale for _, block_scale in quantized_experts])
return (
qweight.reshape(num_experts, n, k // 2),
block_scale.reshape(num_experts, n, k // 16),
qweight,
block_scale,
weight_scale_2,
)