[Quant] Add nvfp4_per_token online MoE quantization (#48538)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-07-16 14:25:27 -07:00
committed by GitHub
parent ab3c1aedf3
commit c95c663049
7 changed files with 285 additions and 5 deletions
+36
View File
@@ -16,7 +16,11 @@ from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PerTensorOnlineLinearMethod,
Fp8PerTensorOnlineMoEMethod,
)
from vllm.model_executor.layers.quantization.online.nvfp4 import (
Nvfp4OnlineMoEMethod,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
@pytest.mark.skipif(
@@ -145,6 +149,38 @@ def test_online_quantization(
print(outputs[0][1])
@pytest.mark.skipif(
not (
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
and has_flashinfer_trtllm_fused_moe()
),
reason="nvfp4_per_token needs a Blackwell (SM100) GPU + FlashInfer TRTLLM MoE.",
)
def test_online_nvfp4_per_token_moe(vllm_runner, monkeypatch) -> None:
"""Online NVFP4 quantizes the MoE and leaves dense layers unquantized."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
with vllm_runner(
"ibm-granite/granite-3.0-1b-a400m-base",
quantization="nvfp4_per_token",
enforce_eager=True,
) as llm:
def check_model(model):
layer = model.model.layers[0]
assert isinstance(
layer.block_sparse_moe.experts._quant_method, Nvfp4OnlineMoEMethod
)
assert isinstance(
layer.self_attn.o_proj.quant_method, UnquantizedLinearMethod
)
llm.apply_model(check_model)
outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)
print(outputs[0][1])
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",