[Quantization] Preserve precision in online NVFP4 expert packing (#50029)
Signed-off-by: S1ro1 <matej.sirovatka@gmail.com>
This commit is contained in:
@@ -9,6 +9,7 @@ from tests.quantization.utils import (
|
||||
_test_online_quant_peak_mem_impl,
|
||||
is_quant_method_supported,
|
||||
)
|
||||
from vllm._custom_ops import scaled_fp4_quant
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerBlockOnlineLinearMethod,
|
||||
@@ -18,6 +19,7 @@ from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.nvfp4 import (
|
||||
Nvfp4OnlineMoEMethod,
|
||||
_quantize_moe_weight_to_nvfp4,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
@@ -184,6 +186,41 @@ def test_online_nvfp4_per_token_moe(vllm_runner, monkeypatch) -> None:
|
||||
print(outputs[0][1])
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
|
||||
),
|
||||
reason="NVFP4 weight quantization needs a Blackwell (SM100) GPU.",
|
||||
)
|
||||
def test_online_nvfp4_quantizes_original_expert_weights() -> None:
|
||||
torch.manual_seed(0)
|
||||
weight = torch.randn(2, 32, 32, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
quantized, block_scale, global_decode_scale = _quantize_moe_weight_to_nvfp4(weight)
|
||||
global_encode_scale = 1.0 / global_decode_scale
|
||||
expected = [
|
||||
scaled_fp4_quant(
|
||||
expert_weight,
|
||||
expert_scale,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
for expert_weight, expert_scale in zip(
|
||||
weight,
|
||||
global_encode_scale,
|
||||
strict=True,
|
||||
)
|
||||
]
|
||||
|
||||
assert torch.equal(
|
||||
quantized,
|
||||
torch.stack([expert_weight for expert_weight, _ in expected]),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_scale,
|
||||
torch.stack([expert_scale for _, expert_scale in expected]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
|
||||
@@ -40,23 +40,34 @@ def _quantize_moe_weight_to_nvfp4(
|
||||
global scale ``(E,)`` stored as ``amax / (fp4_max * fp8_max)``.
|
||||
"""
|
||||
assert weight.dim() == 3, f"expected 3D expert weights, got {weight.shape}"
|
||||
num_experts, n, k = weight.shape
|
||||
k = weight.shape[-1]
|
||||
assert k % 16 == 0, f"last dim must be a multiple of 16, got {k}"
|
||||
|
||||
amax = weight.abs().amax(dim=(1, 2)).to(torch.float32).clamp_min(1e-8)
|
||||
global_scale = (FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX) / amax
|
||||
weight_scale_2 = (1.0 / global_scale).to(torch.float32)
|
||||
|
||||
# scaled_fp4_quant(w, g) == scaled_fp4_quant(w * g, 1), so fold each
|
||||
# expert's scale in and quantize all experts in one call (fp32 to keep the
|
||||
# large scale precise), rather than looping per expert.
|
||||
scaled = (weight.float() * global_scale[:, None, None]).to(weight.dtype)
|
||||
scaled = scaled.reshape(-1, k)
|
||||
one = torch.ones((), device=weight.device, dtype=torch.float32)
|
||||
qweight, block_scale = scaled_fp4_quant(scaled, one, is_sf_swizzled_layout=False)
|
||||
# Keep the original BF16/FP16 values as the quantizer input. Folding each
|
||||
# expert's FP32 global scale into the weight would add a BF16/FP16 rounding
|
||||
# before the group-16 scale and E2M1 values are selected.
|
||||
weight = weight.contiguous()
|
||||
quantized_experts = [
|
||||
scaled_fp4_quant(
|
||||
expert_weight,
|
||||
expert_scale,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
for expert_weight, expert_scale in zip(
|
||||
weight,
|
||||
global_scale,
|
||||
strict=True,
|
||||
)
|
||||
]
|
||||
qweight = torch.stack([quantized for quantized, _ in quantized_experts])
|
||||
block_scale = torch.stack([block_scale for _, block_scale in quantized_experts])
|
||||
return (
|
||||
qweight.reshape(num_experts, n, k // 2),
|
||||
block_scale.reshape(num_experts, n, k // 16),
|
||||
qweight,
|
||||
block_scale,
|
||||
weight_scale_2,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user