[Quantization] Add ModelOpt NVFP4 W4A16 (4-bit weights, fp16/bf16 activations) support (#41769)
Signed-off-by: Juhi Mittal <juhim@nvidia.com>
This commit is contained in:
@@ -239,3 +239,49 @@ def test_modelopt_fp8_pb_wo_checkpoint_setup(default_vllm_config, vllm_runner):
|
||||
output = llm.generate_greedy(["Hello my name is"], max_tokens=4)
|
||||
assert output
|
||||
print(f"ModelOpt FP8_PB_WO output: {output}")
|
||||
|
||||
|
||||
def test_modelopt_nvfp4_config_dispatches_w4a4_method():
|
||||
"""``quant_method="NVFP4"`` (W4A4 default) routes to the existing
|
||||
``ModelOptNvFp4LinearMethod``."""
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4Config,
|
||||
ModelOptNvFp4LinearMethod,
|
||||
)
|
||||
|
||||
config = ModelOptNvFp4Config(
|
||||
quant_method="NVFP4",
|
||||
is_checkpoint_nvfp4_serialized=True,
|
||||
kv_cache_quant_algo=None,
|
||||
exclude_modules=[],
|
||||
)
|
||||
assert config.LinearMethodCls is ModelOptNvFp4LinearMethod
|
||||
assert config.quant_method == "NVFP4"
|
||||
|
||||
|
||||
def test_modelopt_nvfp4_config_dispatches_w4a16_method():
|
||||
"""``quant_method="W4A16_NVFP4"`` routes to the new
|
||||
``ModelOptNvFp4W4A16LinearMethod`` instead of the W4A4 sibling.
|
||||
|
||||
Mirrors the FP8 dispatch precedent (``ModelOptFp8Config`` selects
|
||||
one of three FP8 LinearMethods on ``quant_method``); a regression
|
||||
here would mean a W4A16 NVFP4 checkpoint silently loaded under the
|
||||
W4A4 method, which would try to register an ``input_scale`` runtime
|
||||
parameter and (more importantly) call the cutlass W4A4 NVFP4 GEMM
|
||||
instead of FP4 Marlin.
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4Config,
|
||||
ModelOptNvFp4LinearMethod,
|
||||
ModelOptNvFp4W4A16LinearMethod,
|
||||
)
|
||||
|
||||
config = ModelOptNvFp4Config(
|
||||
quant_method="W4A16_NVFP4",
|
||||
is_checkpoint_nvfp4_serialized=True,
|
||||
kv_cache_quant_algo=None,
|
||||
exclude_modules=[],
|
||||
)
|
||||
assert config.LinearMethodCls is ModelOptNvFp4W4A16LinearMethod
|
||||
assert config.LinearMethodCls is not ModelOptNvFp4LinearMethod
|
||||
assert config.quant_method == "W4A16_NVFP4"
|
||||
|
||||
Reference in New Issue
Block a user