[Quantization][ModelOpt] W4A16 NVFP4 fused MoE + mixed-precision dispatch (#42566)

Signed-off-by: Juhi Mittal <juhim@nvidia.com>
This commit is contained in:
Juhi Mittal
2026-05-22 20:51:49 +00:00
committed by GitHub
parent 08cb46789d
commit e203006a8b
2 changed files with 177 additions and 5 deletions
+140 -1
View File
@@ -6,7 +6,8 @@ Run `pytest tests/quantization/test_modelopt.py`.
"""
import os
from typing import NoReturn
from typing import Any, NoReturn
from unittest.mock import MagicMock, patch
import pytest
import torch
@@ -285,3 +286,141 @@ def test_modelopt_nvfp4_config_dispatches_w4a16_method():
assert config.LinearMethodCls is ModelOptNvFp4W4A16LinearMethod
assert config.LinearMethodCls is not ModelOptNvFp4LinearMethod
assert config.quant_method == "W4A16_NVFP4"
@pytest.mark.parametrize(
"quant_method, expected_use_a16, act_key_is_none",
[
("NVFP4", False, False), # W4A4 default
("W4A16_NVFP4", True, True), # native W4A16 ckpt
],
)
def test_modelopt_nvfp4_moe_dispatches_to_marlin_when_w4a16(
quant_method, expected_use_a16, act_key_is_none
):
"""``ModelOptNvFp4FusedMoE``: when the ckpt's ``quant_method`` is
``W4A16_NVFP4``, the MoE class must pass ``activation_key=None`` to
``select_nvfp4_moe_backend``. That filters out every W4A4 backend
(their ``_supports_quant_scheme`` requires
``(kNvfp4Static, kNvfp4Dynamic)`` exactly); Marlin survives because
it only checks ``weight_key``. A regression here would mean a W4A16
ckpt silently went to the cutlass W4A4 path.
"""
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
ModelOptNvFp4FusedMoE,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kNvfp4Dynamic,
kNvfp4Static,
)
config = ModelOptNvFp4Config(
quant_method=quant_method,
is_checkpoint_nvfp4_serialized=True,
kv_cache_quant_algo=None,
exclude_modules=[],
group_size=16,
)
mock_select = MagicMock(return_value=(MagicMock(), MagicMock()))
with (
patch(
"vllm.model_executor.layers.quantization.modelopt.select_nvfp4_moe_backend",
mock_select,
),
patch(
"vllm.model_executor.layers.quantization.modelopt."
"is_global_sf_supported_for_nvfp4_backend",
return_value=False,
),
):
moe = ModelOptNvFp4FusedMoE(config, MagicMock())
assert moe.use_a16 is expected_use_a16
_, kwargs = mock_select.call_args
assert kwargs["weight_key"] is kNvfp4Static
if act_key_is_none:
assert kwargs["activation_key"] is None
else:
assert kwargs["activation_key"] is kNvfp4Dynamic
@pytest.mark.parametrize(
"per_layer_algo, expected_linear_cls_name",
[
("NVFP4", "ModelOptNvFp4LinearMethod"),
("W4A16_NVFP4", "ModelOptNvFp4W4A16LinearMethod"),
],
)
def test_modelopt_mixed_precision_dispatches_w4a16_layer(
per_layer_algo, expected_linear_cls_name
):
"""``ModelOptMixedPrecisionConfig.get_quant_method`` must route a Linear
layer to the right LinearMethod based on its per-layer ``quant_algo``
entry in ``quantized_layers``. Verifies the new ``W4A16_NVFP4`` branch
coexists with the existing ``NVFP4`` branch without regression. A
regression here would mean a W4A16 layer in a mixed-precision ckpt
silently fell through to ``UnquantizedLinearMethod``.
NOTE: FP8 dispatch (the third branch of get_quant_method) is not
covered here because ``ModelOptFp8LinearMethod.__init__`` reads
``get_current_vllm_config().model_config.dtype``, which requires a
fully constructed ``ModelConfig`` (real model path). FP8 routing in
mixed-precision is exercised by the existing integration tests
above that use the ``vllm_runner`` fixture (e.g.
``test_modelopt_fp8_checkpoint_setup``). Our PR doesn't change the
FP8 branch, so this isn't a coverage gap.
"""
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization import modelopt as m
hf_quant_config: dict[str, Any] = {
"quantization": {
"quant_algo": "MIXED_PRECISION",
"kv_cache_quant_algo": None,
"exclude_modules": [],
"group_size": 16,
"quantized_layers": {
"model.layers.0.fake_proj": {"quant_algo": per_layer_algo},
},
}
}
config = m.ModelOptMixedPrecisionConfig.from_config(hf_quant_config)
fake_layer = MagicMock(spec=LinearBase)
method = config.get_quant_method(fake_layer, "model.layers.0.fake_proj")
expected_cls = getattr(m, expected_linear_cls_name)
assert isinstance(method, expected_cls), (
f"Expected {expected_linear_cls_name}, got {type(method).__name__}"
)
def test_modelopt_mixed_precision_builds_w4a16_sibling_config():
"""Sanity: ``ModelOptMixedPrecisionConfig._from_config`` builds **two**
NVFP4 sub-configs — one for W4A4 (default) and one tagged
``quant_method='W4A16_NVFP4'`` — so per-layer dispatch can hand
Marlin-bound layers the right config without re-instantiating it on
every call.
"""
from vllm.model_executor.layers.quantization import modelopt as m
hf_quant_config: dict[str, Any] = {
"quantization": {
"quant_algo": "MIXED_PRECISION",
"kv_cache_quant_algo": None,
"exclude_modules": [],
"group_size": 16,
"quantized_layers": {
"model.layers.0.a": {"quant_algo": "NVFP4"},
"model.layers.0.b": {"quant_algo": "W4A16_NVFP4"},
},
}
}
config = m.ModelOptMixedPrecisionConfig.from_config(hf_quant_config)
assert config.nvfp4_config.quant_method == "NVFP4"
assert config.nvfp4_config.LinearMethodCls is m.ModelOptNvFp4LinearMethod
assert config.w4a16_nvfp4_config.quant_method == "W4A16_NVFP4"
assert config.w4a16_nvfp4_config.LinearMethodCls is m.ModelOptNvFp4W4A16LinearMethod