From 9c22668436a4d94aab87ea74a220e060415cf1d8 Mon Sep 17 00:00:00 2001 From: Matej Sirovatka <54212263+S1ro1@users.noreply.github.com> Date: Thu, 6 Aug 2026 02:23:54 -0700 Subject: [PATCH] [Quantization] Preserve precision in online NVFP4 expert packing (#50029) Signed-off-by: S1ro1 --- tests/quantization/test_online.py | 37 +++++++++++++++++++ .../layers/quantization/online/nvfp4.py | 31 +++++++++++----- 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index d977cff96d..6a08110774 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -9,6 +9,7 @@ from tests.quantization.utils import ( _test_online_quant_peak_mem_impl, is_quant_method_supported, ) +from vllm._custom_ops import scaled_fp4_quant from vllm.model_executor.layers.linear import UnquantizedLinearMethod from vllm.model_executor.layers.quantization.online.fp8 import ( Fp8PerBlockOnlineLinearMethod, @@ -18,6 +19,7 @@ from vllm.model_executor.layers.quantization.online.fp8 import ( ) from vllm.model_executor.layers.quantization.online.nvfp4 import ( Nvfp4OnlineMoEMethod, + _quantize_moe_weight_to_nvfp4, ) from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe @@ -184,6 +186,41 @@ def test_online_nvfp4_per_token_moe(vllm_runner, monkeypatch) -> None: print(outputs[0][1]) +@pytest.mark.skipif( + not ( + current_platform.is_cuda() and current_platform.is_device_capability_family(100) + ), + reason="NVFP4 weight quantization needs a Blackwell (SM100) GPU.", +) +def test_online_nvfp4_quantizes_original_expert_weights() -> None: + torch.manual_seed(0) + weight = torch.randn(2, 32, 32, device="cuda", dtype=torch.bfloat16) + + quantized, block_scale, global_decode_scale = _quantize_moe_weight_to_nvfp4(weight) + global_encode_scale = 1.0 / global_decode_scale + expected = [ + scaled_fp4_quant( + expert_weight, + expert_scale, + is_sf_swizzled_layout=False, + ) + for expert_weight, expert_scale in zip( + weight, + global_encode_scale, + strict=True, + ) + ] + + assert torch.equal( + quantized, + torch.stack([expert_weight for expert_weight, _ in expected]), + ) + assert torch.equal( + block_scale, + torch.stack([expert_scale for _, expert_scale in expected]), + ) + + @pytest.mark.skipif( not is_quant_method_supported("fp8"), reason="FP8 is not supported on this GPU type.", diff --git a/vllm/model_executor/layers/quantization/online/nvfp4.py b/vllm/model_executor/layers/quantization/online/nvfp4.py index 872643ba26..eaa0069942 100644 --- a/vllm/model_executor/layers/quantization/online/nvfp4.py +++ b/vllm/model_executor/layers/quantization/online/nvfp4.py @@ -40,23 +40,34 @@ def _quantize_moe_weight_to_nvfp4( global scale ``(E,)`` stored as ``amax / (fp4_max * fp8_max)``. """ assert weight.dim() == 3, f"expected 3D expert weights, got {weight.shape}" - num_experts, n, k = weight.shape + k = weight.shape[-1] assert k % 16 == 0, f"last dim must be a multiple of 16, got {k}" amax = weight.abs().amax(dim=(1, 2)).to(torch.float32).clamp_min(1e-8) global_scale = (FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX) / amax weight_scale_2 = (1.0 / global_scale).to(torch.float32) - # scaled_fp4_quant(w, g) == scaled_fp4_quant(w * g, 1), so fold each - # expert's scale in and quantize all experts in one call (fp32 to keep the - # large scale precise), rather than looping per expert. - scaled = (weight.float() * global_scale[:, None, None]).to(weight.dtype) - scaled = scaled.reshape(-1, k) - one = torch.ones((), device=weight.device, dtype=torch.float32) - qweight, block_scale = scaled_fp4_quant(scaled, one, is_sf_swizzled_layout=False) + # Keep the original BF16/FP16 values as the quantizer input. Folding each + # expert's FP32 global scale into the weight would add a BF16/FP16 rounding + # before the group-16 scale and E2M1 values are selected. + weight = weight.contiguous() + quantized_experts = [ + scaled_fp4_quant( + expert_weight, + expert_scale, + is_sf_swizzled_layout=False, + ) + for expert_weight, expert_scale in zip( + weight, + global_scale, + strict=True, + ) + ] + qweight = torch.stack([quantized for quantized, _ in quantized_experts]) + block_scale = torch.stack([block_scale for _, block_scale in quantized_experts]) return ( - qweight.reshape(num_experts, n, k // 2), - block_scale.reshape(num_experts, n, k // 16), + qweight, + block_scale, weight_scale_2, )