[Online quantization] Add online MXFP4 quantization support (#49347)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-08-07 12:39:46 -07:00
committed by GitHub
parent a671679e9f
commit 3518110f2f
18 changed files with 1423 additions and 35 deletions
+34 -3
View File
@@ -14,6 +14,7 @@ from tests.quantization.utils import (
is_quant_method_supported,
)
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm._custom_ops import scaled_fp4_quant
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.online.fp8 import (
@@ -27,6 +28,10 @@ from vllm.model_executor.layers.quantization.online.fp8 import (
_is_tp_sharded,
)
from vllm.model_executor.layers.quantization.online.int8 import Int8OnlineMoEMethod
from vllm.model_executor.layers.quantization.online.mxfp4 import (
Mxfp4OnlineLinearMethod,
Mxfp4OnlineMoEMethod,
)
from vllm.model_executor.layers.quantization.online.nvfp4 import (
Nvfp4OnlineMoEMethod,
_quantize_moe_weight_to_nvfp4,
@@ -40,6 +45,17 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx942, on_gfx950
else:
def on_gfx950() -> bool:
return False
def on_gfx942() -> bool:
return False
DEVICE = current_platform.device_type
@@ -84,6 +100,12 @@ DEVICE = current_platform.device_type
Fp8PerTensorOnlineLinearMethod,
Fp8PerTensorOnlineMoEMethod,
),
(
"mxfp4",
None,
Mxfp4OnlineLinearMethod,
Mxfp4OnlineMoEMethod,
),
],
)
@pytest.mark.parametrize(
@@ -106,8 +128,14 @@ def test_online_quantization(
Does not test performance, peak memory usage, etc.
"""
if use_rocm_aiter:
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
# TODO: Relax this condition once there is a native MXFP4_MXFP4
# linear/moe backend supported on cuda.
if quant_scheme == "mxfp4" and not (on_gfx950() or on_gfx942()):
pytest.skip("mxfp4 online quantization is only tested on AMD gfx942, gfx950.")
if current_platform.is_rocm():
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_rocm_aiter else "0")
rocm_aiter_ops.refresh_env_variables()
if current_platform.is_xpu() and quant_scheme == "fp8_per_block":
pytest.skip("Skip test for online fp8_per_block on XPU platform.")
@@ -144,7 +172,10 @@ def test_online_quantization(
if moe is not None:
assert isinstance(moe._quant_method, expected_moe_cls)
if current_platform.is_cuda() or current_platform.is_xpu():
if quant_scheme == "mxfp4":
# Packed e2m1 values, two per byte.
assert o_proj.weight.dtype == torch.uint8
elif current_platform.is_cuda() or current_platform.is_xpu():
assert o_proj.weight.dtype == torch.float8_e4m3fn
elif current_platform.is_rocm():
assert o_proj.weight.dtype == current_platform.fp8_dtype()