[Quantization] Add ModelOpt NVFP4 W4A16 (4-bit weights, fp16/bf16 activations) support (#41769)

Signed-off-by: Juhi Mittal <juhim@nvidia.com>
This commit is contained in:
Juhi Mittal
2026-05-09 21:15:50 +00:00
committed by GitHub
parent 2ee8c2a56e
commit 7a2b596982
3 changed files with 224 additions and 6 deletions
+46
View File
@@ -239,3 +239,49 @@ def test_modelopt_fp8_pb_wo_checkpoint_setup(default_vllm_config, vllm_runner):
output = llm.generate_greedy(["Hello my name is"], max_tokens=4)
assert output
print(f"ModelOpt FP8_PB_WO output: {output}")
def test_modelopt_nvfp4_config_dispatches_w4a4_method():
"""``quant_method="NVFP4"`` (W4A4 default) routes to the existing
``ModelOptNvFp4LinearMethod``."""
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
ModelOptNvFp4LinearMethod,
)
config = ModelOptNvFp4Config(
quant_method="NVFP4",
is_checkpoint_nvfp4_serialized=True,
kv_cache_quant_algo=None,
exclude_modules=[],
)
assert config.LinearMethodCls is ModelOptNvFp4LinearMethod
assert config.quant_method == "NVFP4"
def test_modelopt_nvfp4_config_dispatches_w4a16_method():
"""``quant_method="W4A16_NVFP4"`` routes to the new
``ModelOptNvFp4W4A16LinearMethod`` instead of the W4A4 sibling.
Mirrors the FP8 dispatch precedent (``ModelOptFp8Config`` selects
one of three FP8 LinearMethods on ``quant_method``); a regression
here would mean a W4A16 NVFP4 checkpoint silently loaded under the
W4A4 method, which would try to register an ``input_scale`` runtime
parameter and (more importantly) call the cutlass W4A4 NVFP4 GEMM
instead of FP4 Marlin.
"""
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
ModelOptNvFp4LinearMethod,
ModelOptNvFp4W4A16LinearMethod,
)
config = ModelOptNvFp4Config(
quant_method="W4A16_NVFP4",
is_checkpoint_nvfp4_serialized=True,
kv_cache_quant_algo=None,
exclude_modules=[],
)
assert config.LinearMethodCls is ModelOptNvFp4W4A16LinearMethod
assert config.LinearMethodCls is not ModelOptNvFp4LinearMethod
assert config.quant_method == "W4A16_NVFP4"