[Quant] Add nvfp4_per_token online MoE quantization (#48538)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -16,7 +16,11 @@ from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
Fp8PerTensorOnlineMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.nvfp4 import (
|
||||
Nvfp4OnlineMoEMethod,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
@@ -145,6 +149,38 @@ def test_online_quantization(
|
||||
print(outputs[0][1])
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
and has_flashinfer_trtllm_fused_moe()
|
||||
),
|
||||
reason="nvfp4_per_token needs a Blackwell (SM100) GPU + FlashInfer TRTLLM MoE.",
|
||||
)
|
||||
def test_online_nvfp4_per_token_moe(vllm_runner, monkeypatch) -> None:
|
||||
"""Online NVFP4 quantizes the MoE and leaves dense layers unquantized."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
with vllm_runner(
|
||||
"ibm-granite/granite-3.0-1b-a400m-base",
|
||||
quantization="nvfp4_per_token",
|
||||
enforce_eager=True,
|
||||
) as llm:
|
||||
|
||||
def check_model(model):
|
||||
layer = model.model.layers[0]
|
||||
assert isinstance(
|
||||
layer.block_sparse_moe.experts._quant_method, Nvfp4OnlineMoEMethod
|
||||
)
|
||||
assert isinstance(
|
||||
layer.self_attn.o_proj.quant_method, UnquantizedLinearMethod
|
||||
)
|
||||
|
||||
llm.apply_model(check_model)
|
||||
outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)
|
||||
print(outputs[0][1])
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
|
||||
Reference in New Issue
Block a user